monitoring

This commit is contained in:
timur 2026-10-03 22:57:52 +05:00
parent de0d48a416
commit 7c6381ee2f
13 changed files with 459 additions and 2 deletions

View File

@ -218,6 +218,7 @@ voice-pipeline importer
voice-pipeline task-import --dry-run
voice-pipeline voice-category
voice-pipeline voice-category-backfill
voice-pipeline metrics-exporter
voice-pipeline crm-backfill --dry-run
voice-pipeline crm-backfill
voice-pipeline reload --list --date 2026-06-23
@ -227,6 +228,29 @@ voice-pipeline reload --repair --date 2026-06-23
voice-pipeline reload --repair --from-date 2026-06-23
```
## Prometheus monitoring
The dedicated exporter reads the pipeline database and exposes queue sizes,
stage statuses, oldest pending ages, and last-event timestamps without changing
call records:
```bash
voice-pipeline metrics-exporter
curl http://127.0.0.1:9101/metrics
```
Its bind address is configured in `[metrics_exporter]`. For a systemd deployment
with an existing Prometheus server, install the service, file-discovery target,
and alert rules with:
```bash
sudo monitoring/install.sh
```
The installer prints the two `prometheus.yml` entries that must be present. It
does not overwrite the existing Prometheus configuration. Service and timer
alerts also require `node_exporter` with its systemd collector enabled.
### Bitrix24 task import: preview and manual run
The safest production check is a dry run. It applies the configured filters,

View File

@ -70,6 +70,12 @@ enabled = true
session_ttl_hours = 12
secure_cookie = true
# Prometheus exporter. Keep it on localhost when Prometheus runs on this host.
[metrics_exporter]
host = "127.0.0.1"
port = 9101
enabled = true
# Bitrix24 OAuth settings are intentionally disabled for now.
# portal_url = "https://company.bitrix24.ru"
# client_id = "local.xxxxxxxxxxxx.xxxxxxxx"

57
monitoring/install.sh Executable file
View File

@ -0,0 +1,57 @@
#!/usr/bin/env bash
# Installs the Voice Pipeline exporter unit and Prometheus discovery/rule files.
# Run as root on the host where Voice Pipeline and Prometheus are installed.
if [ -z "${BASH_VERSION:-}" ]; then
exec bash "$0" "$@"
fi
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
PROMETHEUS_DIR="${PROMETHEUS_DIR:-/etc/prometheus}"
UNIT_DIR="${UNIT_DIR:-/etc/systemd/system}"
APP_DIR="${APP_DIR:-/opt/voice-pipeline}"
if [ "$(id -u)" -ne 0 ]; then
echo "ERROR: run this script as root" >&2
exit 1
fi
echo "==> Updating Voice Pipeline package"
"${APP_DIR}/venv/bin/pip" install --upgrade "${PROJECT_DIR}"
echo "==> Installing metrics exporter service"
install -m 0644 \
"${PROJECT_DIR}/systemd/voice-metrics-exporter.service" \
"${UNIT_DIR}/voice-metrics-exporter.service"
systemctl daemon-reload
systemctl enable --now voice-metrics-exporter.service
echo "==> Installing Prometheus target and alert rules"
install -d -m 0755 "${PROMETHEUS_DIR}/file_sd" "${PROMETHEUS_DIR}/rules.d"
install -m 0644 \
"${SCRIPT_DIR}/voice-pipeline-targets.json" \
"${PROMETHEUS_DIR}/file_sd/voice-pipeline.json"
install -m 0644 \
"${SCRIPT_DIR}/voice-pipeline-alerts.yml" \
"${PROMETHEUS_DIR}/rules.d/voice-pipeline.yml"
PROM_CONFIG="${PROMETHEUS_DIR}/prometheus.yml"
if [ -f "${PROM_CONFIG}" ] && command -v promtool >/dev/null 2>&1; then
promtool check rules "${PROMETHEUS_DIR}/rules.d/voice-pipeline.yml"
fi
echo
echo "Exporter installed: http://127.0.0.1:9101/metrics"
echo "Ensure prometheus.yml contains these entries:"
echo
echo "rule_files:"
echo " - ${PROMETHEUS_DIR}/rules.d/*.yml"
echo
echo "scrape_configs:"
echo " - job_name: voice-pipeline"
echo " file_sd_configs:"
echo " - files: [${PROMETHEUS_DIR}/file_sd/voice-pipeline.json]"
echo
echo "For systemd alerts, node_exporter must expose the systemd collector on :9100."
echo "After checking prometheus.yml, run: systemctl reload prometheus"

View File

@ -0,0 +1,58 @@
groups:
- name: voice-pipeline
rules:
- alert: VoicePipelineExporterDown
expr: up{job="voice-pipeline"} == 0
for: 2m
labels:
severity: critical
annotations:
summary: "Voice Pipeline metrics exporter недоступен"
- alert: VoicePipelineDatabaseDown
expr: voice_pipeline_database_up == 0
for: 2m
labels:
severity: critical
annotations:
summary: "Voice Pipeline не может прочитать PostgreSQL"
- alert: VoicePipelineServiceDown
expr: node_systemd_unit_state{name=~"voice-(importer|dashboard|dashboard-megafon|metrics-exporter)\\.service",state="active"} != 1
for: 2m
labels:
severity: critical
annotations:
summary: "Systemd-сервис {{ $labels.name }} не активен"
- alert: VoicePipelineTimerDown
expr: node_systemd_unit_state{name=~"voice-(collector|analyzer|category|reporter|task-import)\\.timer",state="active"} != 1
for: 5m
labels:
severity: critical
annotations:
summary: "Systemd-таймер {{ $labels.name }} не активен"
- alert: VoicePipelineAnalyzerQueueStuck
expr: voice_pipeline_oldest_pending_age_seconds{stage="analyzer"} > 900
for: 5m
labels:
severity: critical
annotations:
summary: "Очередь анализа {{ $labels.source }} не движется более 15 минут"
- alert: VoicePipelineDownstreamQueueStuck
expr: voice_pipeline_oldest_pending_age_seconds{stage=~"reporter|importer|voice_category"} > 3600
for: 10m
labels:
severity: warning
annotations:
summary: "Очередь {{ $labels.stage }} для {{ $labels.source }} старше часа"
- alert: VoicePipelineDeadLetter
expr: voice_pipeline_calls_status{status="dead_letter"} > 0 or voice_pipeline_stage_status{status="dead_letter"} > 0
for: 1m
labels:
severity: warning
annotations:
summary: "В Voice Pipeline есть необрабатываемые звонки"

View File

@ -0,0 +1,9 @@
[
{
"targets": ["127.0.0.1:9101"],
"labels": {
"job": "voice-pipeline",
"service": "voice-metrics-exporter"
}
}
]

View File

@ -31,6 +31,7 @@ dependencies = [
"fastapi>=0.115,<0.116",
"uvicorn>=0.30,<0.36",
"anyio>=4.4,<4.10",
"prometheus-client>=0.21,<1",
]
[project.optional-dependencies]
@ -81,7 +82,7 @@ voice-pipeline = "voice_pipeline.__main__:main"
[tool.setuptools.packages.find]
where = ["src"]
include = ["voice_pipeline*", "collector*", "analyzer*", "reporter*", "importer*", "task_import*", "voice_category*", "web_dashboard*", "core*"]
include = ["voice_pipeline*", "collector*", "analyzer*", "reporter*", "importer*", "task_import*", "voice_category*", "web_dashboard*", "metrics_exporter*", "core*"]
[tool.setuptools.package-data]
web_dashboard = ["templates/*.j2", "static/*.css", "static/*.js"]

View File

@ -395,6 +395,12 @@ class DashboardConfig(BaseModel):
auth: DashboardAuthConfig = Field(default_factory=DashboardAuthConfig)
class MetricsExporterConfig(BaseModel):
enabled: bool = True
host: str = "127.0.0.1"
port: int = Field(default=9101, ge=1, le=65535)
class LoggingConfig(BaseModel):
level: Literal["DEBUG", "INFO", "WARN", "ERROR", "FATAL"] = "INFO"
@ -426,6 +432,7 @@ class Config(BaseModel):
megafon_dashboard: DashboardConfig = Field(
default_factory=lambda: DashboardConfig(enabled=False, port=8081)
)
metrics_exporter: MetricsExporterConfig = Field(default_factory=MetricsExporterConfig)
logging: LoggingConfig = Field(default_factory=LoggingConfig)
@model_validator(mode="after")

View File

@ -0,0 +1 @@
"""Prometheus metrics exporter for the voice pipeline."""

View File

@ -0,0 +1,182 @@
from __future__ import annotations
import time
from collections import Counter
from collections.abc import Iterable
from datetime import UTC, datetime
from prometheus_client import REGISTRY, start_http_server
from prometheus_client.core import GaugeMetricFamily
from sqlmodel import select
from core.config import load_config
from core.db import get_session, init_db
from core.logger import configure_logging, get_logger
from core.models import CallRecord, CallSource
log = get_logger(__name__)
SOURCES = (CallSource.ASTERISK, CallSource.BITRIX24, CallSource.MEGAFON_ATS)
def _utc_timestamp(value: datetime | None) -> float:
if value is None:
return 0.0
if value.tzinfo is None:
value = value.replace(tzinfo=UTC)
return value.timestamp()
def _age(now: datetime, value: datetime | None) -> float:
timestamp = _utc_timestamp(value)
return max(0.0, now.timestamp() - timestamp) if timestamp else 0.0
class VoicePipelineCollector:
"""Build a fresh, internally consistent DB snapshot for every scrape."""
def collect(self) -> Iterable[GaugeMetricFamily]:
db_up = GaugeMetricFamily(
"voice_pipeline_database_up", "Whether the exporter can query the database"
)
try:
with get_session() as session:
records = list(session.exec(select(CallRecord)).all())
except Exception as exc:
log.error("Metrics database query failed", error=str(exc))
db_up.add_metric([], 0)
yield db_up
return
db_up.add_metric([], 1)
yield db_up
yield from self._record_metrics(records)
def _record_metrics(self, records: list[CallRecord]) -> Iterable[GaugeMetricFamily]:
now = datetime.now(UTC)
calls = GaugeMetricFamily(
"voice_pipeline_calls", "Calls currently present in the database", labels=["source"]
)
statuses = GaugeMetricFamily(
"voice_pipeline_calls_status",
"Calls grouped by pipeline status",
labels=["source", "status"],
)
stages = GaugeMetricFamily(
"voice_pipeline_stage_status",
"Calls grouped by downstream stage status",
labels=["source", "stage", "status"],
)
queue = GaugeMetricFamily(
"voice_pipeline_queue_size",
"Calls waiting or retrying at a processing stage",
labels=["source", "stage"],
)
oldest = GaugeMetricFamily(
"voice_pipeline_oldest_pending_age_seconds",
"Age of the oldest call waiting at a stage",
labels=["source", "stage"],
)
last_event = GaugeMetricFamily(
"voice_pipeline_last_event_timestamp_seconds",
"Timestamp of the latest pipeline event",
labels=["source", "event"],
)
for source in SOURCES:
source_name = str(source)
selected = [record for record in records if record.source == source]
calls.add_metric([source_name], len(selected))
for status, count in Counter(record.status for record in selected).items():
statuses.add_metric([source_name, str(status)], count)
stage_fields = {
"reporter": "reporter_status",
"importer": "importer_status",
"voice_category": "voice_category_status",
}
for stage, field in stage_fields.items():
if stage == "importer" and source == CallSource.MEGAFON_ATS:
continue
for status, count in Counter(str(getattr(r, field)) for r in selected).items():
stages.add_metric([source_name, stage, status], count)
definitions = {
"analyzer": (
[r for r in selected if r.status in {"spooled", "failed"}],
lambda r: r.fetched_at,
),
"reporter": (
[
r
for r in selected
if r.analyzed_at is not None
and r.reporter_status in {"pending", "failed"}
],
lambda r: r.analyzed_at or r.fetched_at,
),
"voice_category": (
[
r
for r in selected
if r.analyzed_at is not None
and r.voice_category_status in {"pending", "failed"}
],
lambda r: r.analyzed_at or r.fetched_at,
),
}
if source != CallSource.MEGAFON_ATS:
definitions["importer"] = (
[
r
for r in selected
if r.analyzed_at is not None
and r.importer_status in {"pending", "processing", "failed"}
],
lambda r: r.analyzed_at or r.fetched_at,
)
for stage, (pending, timestamp_of) in definitions.items():
queue.add_metric([source_name, stage], len(pending))
ages = [_age(now, timestamp_of(record)) for record in pending]
oldest.add_metric([source_name, stage], max(ages, default=0.0))
event_fields = {
"fetched": "fetched_at",
"analyzed": "analyzed_at",
"reported": "reported_at",
"imported": "imported_at",
"categorized": "voice_category_at",
}
for event, field in event_fields.items():
values = [_utc_timestamp(getattr(record, field)) for record in selected]
last_event.add_metric([source_name, event], max(values, default=0.0))
yield calls
yield statuses
yield stages
yield queue
yield oldest
yield last_event
def main() -> None:
cfg = load_config()
configure_logging(level=cfg.logging.level)
if not cfg.metrics_exporter.enabled:
log.info("Metrics exporter disabled by config")
return
init_db()
REGISTRY.register(VoicePipelineCollector())
start_http_server(cfg.metrics_exporter.port, addr=cfg.metrics_exporter.host)
log.info(
"Metrics exporter started",
host=cfg.metrics_exporter.host,
port=cfg.metrics_exporter.port,
)
while True:
time.sleep(3600)
if __name__ == "__main__":
main()

View File

@ -12,6 +12,7 @@ Usage:
voice-pipeline crm-backfill [--dry-run] [--limit N]
voice-pipeline web-dashboard
voice-pipeline web-dashboard-megafon
voice-pipeline metrics-exporter
voice-pipeline reload --reanalyze
[--uniqueid UID | --date YYYY-MM-DD | --status failed|dead_letter]
voice-pipeline reload --recollect [--uniqueid UID | --date YYYY-MM-DD]
@ -95,6 +96,11 @@ def main() -> None:
run()
case "metrics-exporter":
from metrics_exporter.main import main as run
run()
case "reload":
_reload(args[1:])

View File

@ -98,7 +98,8 @@ for unit in \
voice-reporter.timer \
voice-importer.service \
voice-task-import.service \
voice-task-import.timer
voice-task-import.timer \
voice-metrics-exporter.service
do
cp "${SCRIPT_DIR}/${unit}" "${UNIT_DIR}/"
echo " ${unit}"
@ -128,6 +129,7 @@ systemctl enable voice-dashboard-megafon.service
systemctl enable voice-reporter.timer
systemctl enable voice-importer.service
systemctl enable voice-task-import.timer
systemctl enable voice-metrics-exporter.service
# ── Summary ────────────────────────────────────────────────────────────────
echo ""
@ -147,6 +149,7 @@ echo " systemctl start voice-dashboard-megafon.service"
echo " systemctl start voice-reporter.timer"
echo " systemctl start voice-importer.service"
echo " systemctl start voice-task-import.timer"
echo " systemctl start voice-metrics-exporter.service"
echo ""
echo " Check status:"
echo " systemctl status voice-collector.timer"

View File

@ -0,0 +1,35 @@
[Unit]
Description=Voice Pipeline - Prometheus metrics exporter
After=network-online.target postgresql.service
Wants=network-online.target
[Service]
Type=simple
User=voice-pipeline
Group=voice-pipeline
Environment=VOICE_PIPELINE_CONFIG=/etc/voice-pipeline/config.toml
WorkingDirectory=/opt/voice-pipeline
ExecStart=/opt/voice-pipeline/venv/bin/voice-pipeline metrics-exporter
Restart=on-failure
RestartSec=5s
StandardOutput=journal
StandardError=journal
SyslogIdentifier=voice-metrics-exporter
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ReadOnlyPaths=/opt/voice-pipeline /etc/voice-pipeline
ReadWritePaths=/var/lib/voice-pipeline
ProtectHome=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
RestrictRealtime=true
RestrictSUIDSGID=true
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
MemoryMax=128M
CPUQuota=20%
LimitNOFILE=1024
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,68 @@
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from core.models import CallRecord, CallSource
from metrics_exporter.main import VoicePipelineCollector
def _samples(metric):
return {(tuple(sample.labels.items()), sample.value) for sample in metric.samples}
def test_record_metrics_exposes_queue_and_last_event() -> None:
now = datetime.now(UTC)
records = [
CallRecord(
uniqueid="call-1",
source=CallSource.ASTERISK,
status="spooled",
fetched_at=now - timedelta(minutes=20),
),
CallRecord(
uniqueid="call-2",
source=CallSource.ASTERISK,
status="analyzed",
reporter_status="completed",
importer_status="imported",
voice_category_status="completed",
fetched_at=now - timedelta(minutes=10),
analyzed_at=now - timedelta(minutes=5),
),
]
metrics = {
metric.name: metric
for metric in VoicePipelineCollector()._record_metrics(records)
}
queue_samples = _samples(metrics["voice_pipeline_queue_size"])
assert ((('source', 'asterisk'), ('stage', 'analyzer')), 1) in queue_samples
age_samples = _samples(metrics["voice_pipeline_oldest_pending_age_seconds"])
analyzer_ages = [
value
for labels, value in age_samples
if labels == (('source', 'asterisk'), ('stage', 'analyzer'))
]
assert analyzer_ages[0] >= 19 * 60
event_samples = _samples(metrics["voice_pipeline_last_event_timestamp_seconds"])
assert any(
labels == (('source', 'asterisk'), ('event', 'analyzed')) and value > 0
for labels, value in event_samples
)
def test_megafon_has_no_importer_queue() -> None:
record = CallRecord(
uniqueid="megafon_ats:1",
source=CallSource.MEGAFON_ATS,
status="analyzed",
)
metrics = list(VoicePipelineCollector()._record_metrics([record]))
queue = next(metric for metric in metrics if metric.name == "voice_pipeline_queue_size")
assert not any(
sample.labels == {"source": "megafon_ats", "stage": "importer"}
for sample in queue.samples
)