From 5278530467141ec7b2e08c2eba9eac870f25e90f Mon Sep 17 00:00:00 2001 From: patrick Date: Fri, 6 Nov 2026 02:38:10 +0100 Subject: [PATCH] feat: Phase 6b rules with delay, delta format, reminders, zpool default rule Co-Authored-By: Claude Sonnet 5.5 --- CLAUDE.md | 13 +- README.md | 18 ++- app/alerts/__init__.py | 6 +- app/alerts/analyzer.py | 2 +- app/alerts/delivery.py | 30 +++-- app/alerts/delta.py | 62 ++++++++++ app/alerts/formatter.py | 14 ++- app/alerts/gate.py | 50 ++++++++ app/alerts/pipeline.py | 40 +++--- app/alerts/prompts.py | 15 ++- app/alerts/report_status.py | 35 ++++++ app/alerts/repository.py | 15 ++- app/chat/base.py | 4 +- app/chat/matrix.py | 15 +-- app/chat/telegram.py | 6 +- app/checkmk.py | 15 ++- app/constants.py | 9 +- app/db.py | 19 +++ app/models.py | 16 ++- app/poller/events.py | 75 +++++++---- app/poller/loop.py | 9 +- app/poller/options.py | 12 +- app/poller/state.py | 17 ++- app/reminders/__init__.py | 4 + app/reminders/due.py | 25 ++++ app/reminders/worker.py | 53 ++++++++ app/routing/__init__.py | 9 +- app/routing/decisions.py | 43 +++++++ app/routing/limits.py | 9 ++ app/routing/repository.py | 37 ++++-- app/routing/rules.py | 52 +++++--- app/routing/seed.py | 41 +++++++ app/routing/service.py | 6 +- app/routing/validation.py | 37 +++++- app/runtime.py | 21 +++- app/web/alerting_section.py | 21 ++-- app/web/alerts.py | 4 +- app/web/pages.py | 3 +- app/web/problem_status.py | 16 +++ app/web/routing_form.py | 78 ++++++++++++ app/web/routing_pages.py | 66 +--------- app/web/templates/_route_form.html | 16 ++- app/web/templates/alerts.html | 2 +- app/web/templates/base.html | 2 +- app/web/templates/dashboard.html | 5 +- app/web/templates/routing.html | 19 ++- tests/test_adapter_silent.py | 58 +++++++++ tests/test_checkmk_reads.py | 11 +- tests/test_formatter_delta.py | 70 +++++++++++ tests/test_pipeline.py | 5 +- tests/test_pipeline_rules.py | 191 +++++++++++++++++++++++++++++ tests/test_poller.py | 6 +- tests/test_poller_delay.py | 118 ++++++++++++++++++ tests/test_reminders.py | 164 +++++++++++++++++++++++++ tests/test_rule_decisions.py | 124 +++++++++++++++++++ tests/test_rule_seed.py | 60 +++++++++ tests/test_web_routing.py | 9 +- tests/test_web_rules.py | 129 +++++++++++++++++++ 58 files changed, 1781 insertions(+), 230 deletions(-) create mode 100644 app/alerts/delta.py create mode 100644 app/alerts/gate.py create mode 100644 app/alerts/report_status.py create mode 100644 app/reminders/__init__.py create mode 100644 app/reminders/due.py create mode 100644 app/reminders/worker.py create mode 100644 app/routing/decisions.py create mode 100644 app/routing/limits.py create mode 100644 app/routing/seed.py create mode 100644 app/web/problem_status.py create mode 100644 app/web/routing_form.py create mode 100644 tests/test_adapter_silent.py create mode 100644 tests/test_formatter_delta.py create mode 100644 tests/test_pipeline_rules.py create mode 100644 tests/test_poller_delay.py create mode 100644 tests/test_reminders.py create mode 100644 tests/test_rule_decisions.py create mode 100644 tests/test_rule_seed.py create mode 100644 tests/test_web_rules.py diff --git a/CLAUDE.md b/CLAUDE.md index 3d59777..91e81db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,13 +6,13 @@ Vollständiger Plan: `/home/sysops/.claude/plans/ja-alle-bernehmen-eager-lark.md ## Arbeitsregeln - Auf diesem Rechner wird nichts installiert (kein venv, kein pip, keine Pakete). Code nur schreiben, Ausführung und Tests passieren im Ziel-LXC. -- Stand: Phase 1 bis 6 als Code, Phase 7 als Dateien geschrieben, nichts ausgeführt oder getestet (siehe README-Statustabelle). +- Stand: Phase 1 bis 6 und 6b als Code, Phase 7 als Dateien geschrieben, nichts ausgeführt oder getestet (siehe README-Statustabelle). - Code-Kommentare nur bei echtem WHY. ## Festgelegte Entscheidungen - Plattform: Debian-LXC, unprivileged, 1 Core, 1 GB RAM, Python 3, venv im LXC, systemd, Caddy (HTTPS, interne CA). Kein Docker. - Checkmk 2.5 ist die Zielversion (Live-System). Alle REST-Annahmen (Spaltennamen für Services/Hosts, Downtime-Endpunkt, Antwortformat) wurden nach dem Stand von 2.3 formuliert und müssen im ersten Lauf gegen 2.5 geprüft werden (OpenAPI-Doku der Instanz unter `//check_mk/api/1.0/ui/`, Integrationstests Phase 8). Nur REST-API (Automation-User, Lese-Rolle). Kein Webhook, kein Notification-Script, kein eingehender Alarm-Port. -- Poller: 30 s (UI einstellbar), nur Nicht-OK, Zustandsvergleich in SQLite, nur Hard-States, Mindestdauer einstellbar (Default 0 s), Flapping gebündelt. +- Poller: 30 s (UI einstellbar), nur Nicht-OK, Zustandsvergleich in SQLite, nur Hard-States, Verzögerung einstellbar (Default 15 min, Regeln überschreiben, Phase 6b), Flapping gebündelt. - Ollama extern `office.sysops.de:11434`. Zwei Modell-Einstellungen (Alarmtext, Agent), Start `qwen3`, Hermes zum Vergleich. - Chat: Adapter-Interface. Telegram (Long-Polling) und Matrix (direkt über die Client-Server-API mit `httpx`, bewusst nicht `matrix-nio`: kein libolm/E2EE nötig; eigener Homeserver, unverschlüsselter Raum, Bot-User mit Access-Token). Parallel, einzeln abschaltbar, Ziel pro Routing-Regel. - Alarm-Flow: Rohalarm sofort, KI-Analyse per Edit nachgereicht (Telegram editMessage, Matrix m.replace). Max 8 Zeilen DE. Timeout 60 s, 2 Retries, sonst "KI nicht verfügbar". Bündelung pro Host 60 s, max 10 Prompts/min, Recovery ohne KI. Downtime: kein Push, Verlauf "unterdrückt". UI-Stummschaltung mit Ablauf. @@ -74,6 +74,14 @@ tests/ - UI: `/prompts/{name}` (alle), `/prompts/{name}/edit|draft|versions/{n}/activate|test` (Admin), `/settings/retention` (generische Settings-Section mit `extra_template`) und `POST /retention/run`. - Offene Annahmen: Ollama-Retries im Testbereich wie im Betrieb (Request kann bis zu 3 x Timeout dauern); der Setup-Assistent zeigt Aufbewahrung wie Alarmierung als "erledigt", weil kein Pflichtfeld existiert. +## Phase 6b Regeln, Verzögerung, Delta, Erinnerungen (Code geschrieben, ungetestet) +- Migration Index 6: `service_states` + `problem_since`, `reported_at` (Zeitpunkt der letzten Meldung, bei Altzeilen = Migrationszeit), `reported_since` (Beginn des gemeldeten Zustands), `last_reminder_ts` (wird vom Poller-Upsert nie überschrieben); `routes` + `delay_minutes` (NULL = Standard), `output_contains`, `ai_enabled`, `ai_hint`, `reminder_hours` (NULL/0 = keine), `loud`, `use_default_targets`; neue Tabelle `app_state` (key/value, Marker `seed.rule.zfs_zpool`). Bis Index 6 belegt, 6c nimmt Index 7. +- Settings: `alerts.default_delay_minutes` (15, 0-1440), `alerts.recovery_enabled` (an), `alerts.reminder_hours` (0, 0-168). `poll.min_duration` ist entfernt, der Altwert in `settings` bleibt unbeachtet liegen. +- `app/routing/`: `rules.py` (Route, `matches`, `matching_rules` mit stop_on_match, `resolve_targets` inkl. use_default_targets), `decisions.py` (`resolve_delay`, `delay_provider`, `ai_allowed`, `resolve_loud`, `ai_hints` max. 1200, `reminder_interval`), `seed.py` (Zpool-Regel einmalig, Fehler nur geloggt, Aufruf im Lifespan), `limits.py`. Erinnerungen matchen wie Probleme (Art `reminder` nutzt `problem` der Regel). +- Poller: `compute_events(previous, current, now, delay_seconds)` mit Callable pro (Host, Service, Status, Output); Regeln pro Zyklus frisch (`Poller(..., rules=routes.enabled_routes)`). `Event` hat `previous_state`, `previous_duration`, `since_ts`, `problem_since_ts`, `ai_enabled`, `ai_hint`; Art `reminder`. +- Pipeline: `alerts/gate.py` (`ReportGate`: merkt pro Service, ob das Problem gesendet wurde, Fallback aus `alerts` per `report_counts`; Recovery-Schalter und "Recovery nur nach gemeldetem Problem", Gründe `recovery_aus`, `nicht_gemeldet`), Gruppenschlüssel (Host, Ziele, leise); Adapter `send/edit(..., silent=False)`, `Deliverer` übergibt `silent` nur wenn gesetzt. `alerts/delta.py` (Dauerformat, Delta-Texte), `alerts/report_status.py` (Dashboard). +- `app/reminders/`: `due_reminders` (rein) und `ReminderWorker` (60 s, Zeitpunkt wird vor dem Senden gespeichert, im Lifespan gestartet). + ## Weiterer Weg (entschieden) Reihenfolge: Testbasis, erster Lauf im Test-LXC (`pytest`), Korrekturrunde, Phase 6, Phase 6b, Phase 6c, Phase 8, Abnahme, Parallelbetrieb, Tag `v0.1.0`. Verfeinerungen passieren am Live-System, nicht vorab. - Testbasis: `pytest` manuell im Test-LXC (Dev-venv mit `requirements-dev.txt`), dazu `.gitea/workflows/test.yml` für einen späteren Gitea-Actions-Runner. Logging als Zeilenlog nach journald, Level über `CHECKBOT_LOG_LEVEL` (Default INFO), nie Secrets in Logs (Test für die Adapter). Zeitzone über `CHECKBOT_TZ` im Setup-Skript, Default `Europe/Berlin`. @@ -95,6 +103,7 @@ Reihenfolge: Testbasis, erster Lauf im Test-LXC (`pytest`), Korrekturrunde, Phas - Phase 9 (nach der Abnahme, optional, in der UI abschaltbar): Statusberichte nach Zeitplan. Das Modell schreibt aus dem aktuellen Zustand und den Änderungen seit dem letzten Bericht: Zähler (DOWN-Hosts, CRIT, WARN, UNKNOWN), neu, weiter offen, erholt, bestätigt, in Downtime. Der Zeitplan ist in der UI einstellbar: mehrere Uhrzeiten, Wochentage, Zeitzone des LXC, Ziele, Inhalt voll oder kurz. Dazu ein Button "Bericht jetzt senden" und der Befehl `/report`. Der Bericht merkt sich den Stand des letzten Berichts (Vergleich "seit 18:05"). Verfeinerung am Live-System. - Delta in der Alarmnachricht (Phase 6b): `Event` bekommt Vorzustand und Dauer. Alarmzeile zeigt Übergang und Dauer (`WARN → CRIT, seit 18:05 (vorher WARN seit 2 h)`), Recovery die Gesamtdauer (`wieder OK nach 3 h 12 min (zuletzt CRIT)`), Erinnerungen `weiter CRIT seit 4 h`. - Recovery-Meldungen sind in der UI schaltbar: globaler Schalter `alerts.recovery_enabled` (Default an) in der Section "Alarmierung" und pro Regel über die Ereignisarten (`kinds` enthält `recovery` oder nicht). Greift zusätzlich zur Regel "Recovery nur, wenn das Problem vorher gemeldet wurde". Ist Recovery aus, bleibt sie im Verlauf sichtbar, wird aber nicht gesendet. +- Verzögerung (Phase 6b): Section "Alarmierung" bekommt `alerts.default_delay_minutes` (Werkseinstellung 15; ersetzt `poll.min_duration`). Jede Regel kann die Verzögerung überschreiben, auch mit 0. Passen mehrere Regeln, gilt die kleinste. Sie zählt ab der letzten Statusänderung, eine Verschlechterung startet sie neu; wird das Problem vorher OK, gibt es weder Alarm noch Recovery. Das Dashboard zeigt wartende Probleme ("wartet noch 8 min"). Mute und Downtime wirken auch während der Wartezeit. Host-Alarme und der Checkmk-Ausfall (Phase 6c) kommen immer sofort, Host-Verzögerung `alerts.host_delay_minutes` Standard 0. Die vorinstallierte Regel "ZFS Zpool" (`*zpool*`) hat Verzögerung 0, damit gilt ab Werk: Zpool sofort, alles andere nach 15 min. - Aktionen: Der Bot führt nie Shell-Befehle aus (weder lokal noch per SSH). Webhook pro Regel (POST JSON an Allowlist-URLs, Audit) kommt erst nach der Abnahme. - Phase 8: Integrationstests mit Marker `integration` (`pytest -m integration`), Zugangsdaten per Umgebung, nur lesend; Downtime nur mit ausdrücklichem Flag und gegen einen Testhost. Testhost `checkbot-test` in Checkmk mit schaltbarem lokalem Check plus halbautomatisches `tests/e2e/run.py`. - Abnahme (`docs/ACCEPTANCE.md`): CRIT in Telegram und Element, KI-Analyse binnen 90 s, Recovery ohne KI, Downtime und `mute` unterdrücken, Ollama aus ergibt Rohalarm mit "KI nicht verfügbar", Neustart ohne Doppelalarme, Backup und Restore real getestet, `/health` im Checkmk grün, Baseline-Zusammenfassung beim Erstlauf, Host-down unterdrückt Service-Alarme, Meldung bei Checkmk-Ausfall, 24 h Dauerlauf ohne Fehler im Journal. diff --git a/README.md b/README.md index b850238..d6fda35 100644 --- a/README.md +++ b/README.md @@ -8,20 +8,29 @@ Bot für Checkmk. Er liest den Zustand per REST-API, lässt Probleme von einem e ### Alarme melden - Fragt Checkmk alle 30 s ab (einstellbar) und erkennt neue Probleme, Statuswechsel (WARN/CRIT/UNKNOWN), Recoveries und Flapping. -- Meldet nur Hard-States, optional erst nach einer Mindestdauer. +- Meldet nur Hard-States und erst nach einer Verzögerung (siehe unten). - Respektiert Downtimes: Services in Downtime lösen keine Nachricht aus und erscheinen im Verlauf als "unterdrückt". - Bündelt Alarme eines Hosts zu einer Nachricht und begrenzt den Durchsatz, damit eine Alarmflut den Chat und Ollama nicht überrollt. - Fällt Ollama aus, kommt der Rohalarm trotzdem sofort. Die Analyse wird nachgereicht oder als "KI nicht verfügbar" markiert. +### Verzögerung, Delta, Erinnerungen und Recovery +- Verzögerung: Ein Problem wird erst gemeldet, wenn sein jetziger Zustand seit der Standard-Verzögerung (Alarmierung, Werkseinstellung 15 min, 0 = sofort) besteht; eine Regel kann das überschreiben, auch mit 0. Die Zeit zählt ab der letzten Statusänderung in Checkmk, eine Verschlechterung startet sie neu. Wird das Problem vorher wieder OK, gibt es weder Alarm noch Recovery. Mute und Downtime wirken auch in der Wartezeit; was bei Ablauf der Verzögerung unterdrückt ist, wird nicht nachgeholt. Die frühere Einstellung `poll.min_duration` gibt es nicht mehr, ein vorhandener Altwert wird ignoriert. +- Delta in der Nachricht: `WARN → CRIT, seit 18:05 (vorher WARN seit 2 h)`, bei Recovery `wieder OK nach 3 h 12 min (zuletzt CRIT)`, bei Erinnerungen `weiter CRIT seit 4 h: `. Zeiten in der lokalen Zeit des Prozesses. +- Erinnerungen: nur für offene CRIT-Probleme, die gemeldet wurden und weiter CRIT sind, nicht bei Mute, Downtime oder Flapping. Intervall aus den passenden Regeln (kleinstes), sonst die Standard-Erinnerung in Stunden (Alarmierung, Werkseinstellung 0 = aus). Gemessen ab der letzten Meldung bzw. Erinnerung, der Zeitpunkt ist in der Datenbank gespeichert (kein Doppelversand nach Neustart). Erinnerungen laufen ohne KI durch dasselbe Routing wie Alarme und stehen im Verlauf als Art `reminder`. +- Recovery: Schalter "Recovery-Meldungen senden" (Alarmierung, Werkseinstellung an) und pro Regel über die Alarmarten. Eine Recovery wird nur gesendet, wenn das Problem vorher tatsächlich gemeldet wurde; sonst (oder bei ausgeschaltetem Schalter) steht sie unterdrückt im Verlauf. + ### KI-Analyse - Pro Problem kurze Analyse auf Deutsch (max. 8 Zeilen): Zusammenfassung, wahrscheinliche Ursache, nächster Prüfschritt. - Kontext: Host, Service, Plugin-Output, letzte Statuswechsel, Downtime-Info. +- Ein KI-Hinweis der Regel steht als Abschnitt "Zusatzkontext des Administrators" im Nutzer-Prompt vor dem Datenblock (bereinigt, gekürzt, ohne Begrenzer). Der feste Schutzhinweis im System-Prompt bleibt unverändert. - Plugin-Output gilt als nicht vertrauenswürdig. Er wird abgegrenzt, gekürzt und bereinigt, und die KI kann nichts auslösen. ### Chat (Telegram und Element) - Beide Backends parallel, einzeln abschaltbar. - Matrix läuft direkt über die Client-Server-API (`httpx`, kein `matrix-nio`, keine E2EE-Abhängigkeiten). Der Alarmraum muss unverschlüsselt sein, der Bot-User braucht einen Access-Token und muss dem Raum beigetreten sein. Die Analyse kommt per `m.replace`-Edit. -- Routing (UI `/routing`): Regeln mit Host-/Service-Glob, Mindest-Schweregrad, Alarmarten und Zielen (Backend plus Chat-ID bzw. Raum-ID). Regeln gelten von oben nach unten, jede passende fügt ihre Ziele hinzu, optional mit Stopp nach Treffer. Passt keine Regel, gehen Alarme an die Standardziele aller aktivierten Backends. Alarme eines Hosts mit gleicher Zielmenge werden zu einer Nachricht gebündelt. +- Regeln (UI `/routing`, in der Navigation "Regeln"): Host-/Service-Glob, optional "Ausgabe enthält" (Teilstring, keine Regex), Mindest-Schweregrad, Alarmarten und Ziele (Backend plus Chat-ID bzw. Raum-ID). Regeln gelten von oben nach unten, jede passende fügt ihre Ziele hinzu, optional mit Stopp nach Treffer. "Standardziele" in einer Regel fügt die Standardziele aller aktivierten Backends zu den expliziten Zielen hinzu. Eine passende Regel ganz ohne Ziele und ohne Standardziele sendet bewusst nichts. Passt keine Regel, gehen Alarme an die Standardziele. Alarme eines Hosts mit gleicher Zielmenge und gleicher Lautstärke werden zu einer Nachricht gebündelt. +- Weitere Regeloptionen: Verzögerung in Minuten (leer = Standard), KI an/aus, KI-Hinweis (max. 800 Zeichen), Erinnerung bei CRIT in Stunden (leer oder 0 = keine) und laut/leise (leise: Telegram ohne Benachrichtigung, Matrix als `m.notice`). Mehrere passende Regeln: die kleinste Verzögerung und das kleinste Erinnerungsintervall gelten, KI ist aus, sobald eine passende Regel sie ausschaltet, laut gilt, sobald eine passende Regel laut ist, KI-Hinweise werden in Regelreihenfolge zusammengefügt (insgesamt max. 1200 Zeichen). +- Vorinstallierte Regel "ZFS Zpool" (einmalig beim ersten Start angelegt, danach editier- und löschbar, eine gelöschte Regel kommt nicht zurück): Service `*zpool*` ab WARN, Standardziele, Verzögerung 0, KI mit Hinweis auf typische Prüfschritte, Erinnerung bei CRIT alle 4 h. - Stummschaltung (UI `/mutes`): Host-/Service-Glob mit Ablauf (30 min bis 24 h oder freie Minuten) und Grund. Passende Probleme, Flapping und Recoveries werden nicht gemeldet und im Verlauf als unterdrückt (Stummschaltung) geführt. Abgelaufene Stummschaltungen räumt der Bot selbst auf. - Der Versand läuft über eine Warteschlange (max. 1000 Events, bei Überlauf wird das älteste verworfen), ein langsames Chat-Backend bremst den Poller nicht. - Rohalarm zuerst, die Analyse ersetzt oder ergänzt die Nachricht per Edit. @@ -70,9 +79,9 @@ Bot für Checkmk. Er liest den Zustand per REST-API, lässt Probleme von einem e ### Web-UI - Anmeldung mit lokalen Nutzern (Argon2), optional TOTP. Rollen `admin` und `viewer`. -- Einstellungen mit Verbindungstest: Checkmk, Ollama, Telegram, Matrix, Checkmk Downtime. Dazu die Bereiche Alarmierung (Abfrageintervall, Mindestdauer, Bündelungsfenster, KI-Analysen pro Minute) und Aufbewahrung. +- Einstellungen mit Verbindungstest: Checkmk, Ollama, Telegram, Matrix, Checkmk Downtime. Dazu die Bereiche Alarmierung (Abfrageintervall, Standard-Verzögerung, Recovery-Schalter, Standard-Erinnerung, Bündelungsfenster, KI-Analysen pro Minute) und Aufbewahrung. - Setup-Assistent beim ersten Start, Dashboard mit Live-Status, Alarmverlauf, Audit-Log, Benutzerverwaltung. -- Routing und Stummschaltungen (Admins bearbeiten, Viewer sehen). +- Regeln und Stummschaltungen (Admins bearbeiten, Viewer sehen). Das Dashboard zeigt bei offenen Problemen den Meldestatus: "gemeldet", "wartet noch N min" oder "unterdrückt (Grund)". - Prompts (UI `/prompts`), siehe unten. Aufbewahrung (UI `/settings/retention`), siehe unten. - Tokens und Secrets liegen verschlüsselt in der Datenbank und werden in der UI nie wieder angezeigt. @@ -99,6 +108,7 @@ Web-UI (FastAPI, Jinja2) ---- SQLite (Einstellungen, Verlauf, Audit, Nutzer) | Phase 4: Matrix-Adapter, Routing, Stummschaltung in der UI | Code geschrieben, ungetestet | | Phase 5: Befehle, `mute`/`downtime`, Agent-Loop, Listener, Agent-Ansicht | Code geschrieben, ungetestet | | Phase 6: Prompt-Versionierung mit Rollback und Testbereich, Retention | Code geschrieben, ungetestet | +| Phase 6b: Regeln (Ausgabefilter, Verzögerung, KI-Hinweis, Erinnerung, laut/leise), Delta, Erinnerungen, Recovery-Schalter, Zpool-Standardregel | Code geschrieben, ungetestet | | Phase 7: Deployment (`setup-lxc.sh`, Caddy, systemd, Backup, Checkmk-Local-Check) | Dateien geschrieben, nicht auf einem System ausprobiert | | Phase 8: Integrationstests | geplant | diff --git a/app/alerts/__init__.py b/app/alerts/__init__.py index 6b05525..bf94cfc 100644 --- a/app/alerts/__init__.py +++ b/app/alerts/__init__.py @@ -3,15 +3,17 @@ from .bundler import HostBundler from .delivery import Deliverer from .extras import AlertActions, Extras from .formatter import Analysis, format_message +from .gate import ReportGate from .pipeline import AlertPipeline from .prompts import (DATA_NOTICE as ALERT_DATA_NOTICE, DEFAULT_ALERT_SYSTEM_PROMPT, build_messages, build_system_prompt, build_user_prompt, sanitize) from .ratelimit import SlidingWindowLimiter +from .report_status import ReportStatus, report_status from .repository import AlertRepository __all__ = [ "ALERT_DATA_NOTICE", "DEFAULT_ALERT_SYSTEM_PROMPT", "alert_model", "build_messages", "AlertActions", "AlertAnalyzer", "AlertPipeline", "AlertRepository", "Analysis", "AnalysisResult", "Deliverer", "Extras", - "HostBundler", "SlidingWindowLimiter", "build_system_prompt", "build_user_prompt", "format_message", - "sanitize", + "HostBundler", "ReportGate", "ReportStatus", "SlidingWindowLimiter", "build_system_prompt", + "build_user_prompt", "format_message", "report_status", "sanitize", ] diff --git a/app/alerts/analyzer.py b/app/alerts/analyzer.py index 766ab79..a388c08 100644 --- a/app/alerts/analyzer.py +++ b/app/alerts/analyzer.py @@ -59,7 +59,7 @@ class AlertAnalyzer: async def _ask(self, client: OllamaClient, event: Event) -> AnalysisResult: history = await asyncio.to_thread(self._history, event.host, event.service, RECENT_CHANGES_DEFAULT) editable = await asyncio.to_thread(self._editable_prompt) - messages = build_messages(editable, event, history) + messages = build_messages(editable, event, history, event.ai_hint) model = alert_model(self._store) try: reply = await client.chat(model, messages) diff --git a/app/alerts/delivery.py b/app/alerts/delivery.py index ede95e6..2150af6 100644 --- a/app/alerts/delivery.py +++ b/app/alerts/delivery.py @@ -18,24 +18,26 @@ class Deliverer: self._adapters = adapters self._timeout = timeout - async def send(self, text: str, targets: Sequence[Target], - extras: ExtrasProvider | None = None) -> list[MessageRef]: - results = await asyncio.gather(*(self._send_one(target, text, extras) for target in targets)) + async def send(self, text: str, targets: Sequence[Target], extras: ExtrasProvider | None = None, + silent: bool = False) -> list[MessageRef]: + results = await asyncio.gather(*(self._send_one(target, text, extras, silent) for target in targets)) return [ref for ref in results if ref is not None] - async def edit(self, refs: Sequence[MessageRef], text: str, extras: ExtrasProvider | None = None) -> None: - await asyncio.gather(*(self._edit_one(ref, text, extras) for ref in refs)) + async def edit(self, refs: Sequence[MessageRef], text: str, extras: ExtrasProvider | None = None, + silent: bool = False) -> None: + await asyncio.gather(*(self._edit_one(ref, text, extras, silent) for ref in refs)) - async def _send_one(self, target: Target, text: str, extras: ExtrasProvider | None) -> MessageRef | None: - full, options = _apply(text, extras(target.backend) if extras else None) + async def _send_one(self, target: Target, text: str, extras: ExtrasProvider | None, + silent: bool) -> MessageRef | None: + full, options = _apply(text, extras(target.backend) if extras else None, silent) try: return await self._call(target.backend, lambda adapter: adapter.send(target.target, full, **options)) except Exception as exc: log.warning("Senden über %s fehlgeschlagen: %s", target.backend, _brief(exc)) return None - async def _edit_one(self, ref: MessageRef, text: str, extras: ExtrasProvider | None) -> None: - full, options = _apply(text, extras(ref.backend) if extras else None) + async def _edit_one(self, ref: MessageRef, text: str, extras: ExtrasProvider | None, silent: bool) -> None: + full, options = _apply(text, extras(ref.backend) if extras else None, silent) try: await self._call(ref.backend, lambda adapter: adapter.edit(ref, full, **options)) except Exception as exc: @@ -58,11 +60,13 @@ class Deliverer: return await asyncio.wait_for(action(fresh), self._timeout) -def _apply(text: str, extra: Extras | None) -> tuple[str, dict]: +def _apply(text: str, extra: Extras | None, silent: bool = False) -> tuple[str, dict]: + options: dict = {"silent": True} if silent else {} if extra is None: - return text, {} - full = f"{text}\n{extra.hint}" if extra.hint else text - return full, ({"buttons": extra.buttons} if extra.buttons else {}) + return text, options + if extra.buttons: + options["buttons"] = extra.buttons + return (f"{text}\n{extra.hint}" if extra.hint else text), options def _brief(exc: Exception) -> str: diff --git a/app/alerts/delta.py b/app/alerts/delta.py new file mode 100644 index 0000000..4d695a9 --- /dev/null +++ b/app/alerts/delta.py @@ -0,0 +1,62 @@ +from datetime import datetime + +from ..models import KIND_PROBLEM, Event, state_name + +MINUTE = 60 +HOUR = 3600 +DAY = 86400 +CLOCK_FORMAT = "%H:%M" +DATE_CLOCK_FORMAT = "%d.%m. %H:%M" +ARROW = "→" +LESS_THAN_MINUTE = "unter 1 min" + + +def format_duration(seconds: int) -> str: + seconds = max(int(seconds), 0) + if seconds < MINUTE: + return LESS_THAN_MINUTE + if seconds < HOUR: + return f"{seconds // MINUTE} min" + if seconds < DAY: + hours, minutes = seconds // HOUR, (seconds % HOUR) // MINUTE + return f"{hours} h {minutes} min" if minutes else f"{hours} h" + days, rest = divmod(seconds, DAY) + hours = rest // HOUR + return f"{days} d {hours} h" if hours else f"{days} d" + + +def format_clock(ts: int, now: int) -> str: + moment = datetime.fromtimestamp(ts) + same_day = moment.date() == datetime.fromtimestamp(now).date() + return moment.strftime(CLOCK_FORMAT if same_day else DATE_CLOCK_FORMAT) + + +def state_label(event: Event) -> str: + if event.previous_state is not None and event.previous_state != event.state and event.kind == KIND_PROBLEM: + return f"{state_name(event.previous_state)} {ARROW} {state_name(event.state)}" + return state_name(event.state) + + +def since_text(event: Event) -> str: + if event.since_ts is None: + return "" + text = f"seit {format_clock(event.since_ts, event.ts)}" + if event.previous_state is not None and event.previous_duration is not None: + text += f" (vorher {state_name(event.previous_state)} seit {format_duration(event.previous_duration)})" + return text + + +def recovery_text(event: Event) -> str: + text = "wieder OK" + if event.problem_since_ts is not None: + text += f" nach {format_duration(event.ts - event.problem_since_ts)}" + if event.previous_state is not None: + text += f" (zuletzt {state_name(event.previous_state)})" + return text + + +def reminder_text(event: Event) -> str: + text = f"weiter {state_name(event.state)}" + if event.since_ts is not None: + text += f" seit {format_duration(event.ts - event.since_ts)}" + return text diff --git a/app/alerts/formatter.py b/app/alerts/formatter.py index 5fdc801..3775e80 100644 --- a/app/alerts/formatter.py +++ b/app/alerts/formatter.py @@ -3,7 +3,8 @@ from dataclasses import dataclass from html import escape from typing import Mapping, Sequence -from ..models import KIND_FLAPPING, KIND_RECOVERY, Event, state_name +from ..models import KIND_FLAPPING, KIND_RECOVERY, KIND_REMINDER, Event, state_name +from .delta import reminder_text, recovery_text, since_text, state_label TELEGRAM_LIMIT = 4096 OUTPUT_MAX = 300 @@ -11,7 +12,6 @@ AI_MAX_LINES = 8 AI_LINE_MAX = 300 ELLIPSIS = "…" AI_UNAVAILABLE = "KI nicht verfügbar" -RECOVERY_TEXT = "wieder OK" THINK_RE = re.compile(r".*?", re.DOTALL | re.IGNORECASE) @@ -34,15 +34,19 @@ def clip_ai_text(text: str) -> str: def _label(event: Event) -> str: if event.kind == KIND_FLAPPING: return f"FLAPPING ({state_name(event.state)})" - return state_name(event.state) + return state_label(event) def _event_line(event: Event) -> str: + since = since_text(event) if event.kind not in (KIND_RECOVERY, KIND_REMINDER, KIND_FLAPPING) else "" if event.kind == KIND_RECOVERY: - detail = RECOVERY_TEXT + detail = recovery_text(event) + elif event.kind == KIND_REMINDER: + detail = f"{reminder_text(event)}: {clip(event.output, OUTPUT_MAX)}" else: detail = clip(event.output, OUTPUT_MAX) - return f"{escape(_label(event))} {escape(event.service)}: {escape(detail)}" + head = f"{escape(_label(event))} {escape(event.service)}" + return f"{head}, {escape(since)}: {escape(detail)}" if since else f"{head}: {escape(detail)}" def _analysis_block(event: Event, analysis: Analysis, single: bool) -> str: diff --git a/app/alerts/gate.py b/app/alerts/gate.py new file mode 100644 index 0000000..b76c140 --- /dev/null +++ b/app/alerts/gate.py @@ -0,0 +1,50 @@ +import asyncio +from dataclasses import replace +from typing import Callable + +from ..models import (KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, KIND_REMINDER, SUPPRESS_NOT_REPORTED, + SUPPRESS_RECOVERY_OFF, Event, Key) + +REPORTED_KINDS = (KIND_PROBLEM, KIND_FLAPPING, KIND_REMINDER) + +ReportCounts = Callable[[str, str], tuple[int, int]] + + +class ReportGate: + """Merkt pro Service, ob das laufende Problem tatsächlich gesendet wurde (Grundlage für Recovery).""" + + def __init__(self, counts: ReportCounts): + self._counts = counts + self._reported: dict[Key, bool] = {} + + async def apply(self, event: Event, recovery_enabled: bool) -> Event: + if event.kind != KIND_RECOVERY or event.suppressed: + return event + if not recovery_enabled: + return replace(event, suppressed=True, suppress_reason=SUPPRESS_RECOVERY_OFF) + if not await self._was_reported((event.host, event.service)): + return replace(event, suppressed=True, suppress_reason=SUPPRESS_NOT_REPORTED) + return event + + async def track(self, event: Event, sent: bool) -> None: + key = (event.host, event.service) + if event.kind == KIND_RECOVERY: + self._reported.pop(key, None) + elif event.kind in REPORTED_KINDS: + before = await self._sent_before(key) + self._reported[key] = before or sent + + async def _sent_before(self, key: Key) -> bool: + known = self._reported.get(key) + if known is not None: + return known + _total, sent = await asyncio.to_thread(self._counts, *key) + return sent > 0 + + async def _was_reported(self, key: Key) -> bool: + known = self._reported.get(key) + if known is not None: + return known + total, sent = await asyncio.to_thread(self._counts, *key) + # Ohne jede Historie (z. B. durch Retention gelöscht) wird die Recovery nicht verschluckt. + return total == 0 or sent > 0 diff --git a/app/alerts/pipeline.py b/app/alerts/pipeline.py index 9cb9105..014ad65 100644 --- a/app/alerts/pipeline.py +++ b/app/alerts/pipeline.py @@ -2,18 +2,19 @@ import asyncio import logging import time import uuid -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Callable, Protocol from ..chat import MessageRef, Target from ..models import AI_DONE, AI_FAILED, AI_SKIPPED, Event from ..mutes import Mute, apply_mutes -from ..routing import Route, resolve_targets +from ..routing import Route, ai_allowed, ai_hints, resolve_loud, resolve_targets from .analyzer import AlertAnalyzer from .bundler import HostBundler from .delivery import AdapterProvider, Deliverer from .extras import AlertActions from .formatter import Analysis, format_message +from .gate import ReportGate from .repository import AlertRepository log = logging.getLogger(__name__) @@ -23,6 +24,8 @@ PRUNE_INTERVAL_SECONDS = 60.0 QUEUE_MAX = 1000 DRAIN_TIMEOUT_SECONDS = 15.0 +GroupKey = tuple[str, tuple[Target, ...], bool] + class RouteSource(Protocol): def enabled_routes(self) -> list[Route]: ... @@ -39,13 +42,15 @@ class Dispatch: events: list[Event] alert_ids: list[int] refs: list[MessageRef] + silent: bool = False class AlertPipeline: def __init__(self, repo: AlertRepository, adapters: AdapterProvider, analyzer: AlertAnalyzer, bundle_seconds: Callable[[], int], clock: Callable[[], float] = time.time, *, routes: RouteSource | None = None, mutes: MuteSource | None = None, - actions: AlertActions | None = None, queue_size: int = QUEUE_MAX): + actions: AlertActions | None = None, queue_size: int = QUEUE_MAX, + recovery_enabled: Callable[[], bool] = lambda: True): self._repo = repo self._adapters = adapters self._deliverer = Deliverer(adapters) @@ -55,6 +60,8 @@ class AlertPipeline: self._routes = routes self._mutes = mutes self._actions = actions + self._recovery_enabled = recovery_enabled + self._gate = ReportGate(repo.report_counts) self._bundler = HostBundler() self._queue: asyncio.Queue[Event] = asyncio.Queue(maxsize=queue_size) self._tasks: set[asyncio.Task] = set() @@ -93,12 +100,12 @@ class AlertPipeline: log.exception("Gebündelte Alarme konnten beim Beenden nicht gesendet werden") async def flush_due(self) -> None: - for (_host, targets), batch in self._bundler.due(self._clock(), self._bundle_seconds()): - await self._dispatch(batch, targets) + for (_host, targets, silent), batch in self._bundler.due(self._clock(), self._bundle_seconds()): + await self._dispatch(batch, targets, silent) async def flush_all(self) -> None: - for (_host, targets), batch in self._bundler.take_all(): - await self._dispatch(batch, targets) + for (_host, targets, silent), batch in self._bundler.take_all(): + await self._dispatch(batch, targets, silent) async def wait_idle(self) -> None: while self._tasks: @@ -156,12 +163,17 @@ class AlertPipeline: mutes = await asyncio.to_thread(self._mutes.active, now) if self._mutes else [] routes = await asyncio.to_thread(self._routes.enabled_routes) if self._routes else [] defaults = [Target(a.name, a.default_target) for a in await self._adapters()] - groups: dict[tuple[str, tuple[Target, ...]], list[Event]] = {} + recovery_enabled = self._recovery_enabled() + groups: dict[GroupKey, list[Event]] = {} stored_only: list[Event] = [] for event in apply_mutes(events, mutes, now): + event = await self._gate.apply(event, recovery_enabled) + event = replace(event, ai_enabled=ai_allowed(routes, event), ai_hint=ai_hints(routes, event)) targets = () if event.suppressed else tuple(sorted(resolve_targets(routes, event, defaults))) + await self._gate.track(event, bool(targets)) if targets: - groups.setdefault((event.host, targets), []).append(event) + key = (event.host, targets, not resolve_loud(routes, event)) + groups.setdefault(key, []).append(event) else: stored_only.append(event) if stored_only: @@ -170,7 +182,7 @@ class AlertPipeline: for key, group in groups.items(): batch = self._bundler.submit(key, group, now, window) if batch: - await self._dispatch(batch, key[1]) + await self._dispatch(batch, key[1], key[2]) async def _store_only(self, events: list[Event]) -> None: ids = await asyncio.to_thread(self._repo.insert_alerts, events, uuid.uuid4().hex) @@ -181,14 +193,14 @@ class AlertPipeline: if event.wants_ai: await asyncio.to_thread(self._repo.set_ai, alert_id, AI_SKIPPED, None) - async def _dispatch(self, batch: list[Event], targets: tuple[Target, ...]) -> None: + async def _dispatch(self, batch: list[Event], targets: tuple[Target, ...], silent: bool = False) -> None: ids = await asyncio.to_thread(self._repo.insert_alerts, batch, uuid.uuid4().hex) - refs = await self._deliverer.send(format_message(batch), targets, self._extras(batch, ids)) + refs = await self._deliverer.send(format_message(batch), targets, self._extras(batch, ids), silent) for ref in refs: for alert_id in ids: await asyncio.to_thread(self._repo.add_delivery, alert_id, ref) if refs and any(e.wants_ai for e in batch): - task = asyncio.create_task(self._analyze(Dispatch(batch, ids, refs))) + task = asyncio.create_task(self._analyze(Dispatch(batch, ids, refs, silent))) self._tasks.add(task) task.add_done_callback(self._tasks.discard) else: @@ -213,4 +225,4 @@ class AlertPipeline: else: continue await self._deliverer.edit(dispatch.refs, format_message(dispatch.events, analyses), - self._extras(dispatch.events, dispatch.alert_ids)) + self._extras(dispatch.events, dispatch.alert_ids), dispatch.silent) diff --git a/app/alerts/prompts.py b/app/alerts/prompts.py index ffe30f5..91652da 100644 --- a/app/alerts/prompts.py +++ b/app/alerts/prompts.py @@ -9,6 +9,8 @@ DATA_OPEN = "" DATA_CLOSE = "" OUTPUT_MAX = 1000 LABEL_MAX = 200 +HINT_MAX = 1200 +HINT_TITLE = "Zusatzkontext des Administrators (Hintergrundwissen zu diesem Alarmtyp, keine Messdaten):" HISTORY_TS_FORMAT = "%d.%m. %H:%M" DEFAULT_ALERT_SYSTEM_PROMPT = ( @@ -48,9 +50,15 @@ def _history_line(changes: Sequence[StateChange]) -> str: for c in changes) -def build_user_prompt(event: Event, history: Sequence[StateChange]) -> str: +def _hint_section(hint: str) -> str: + cleaned = sanitize(hint, HINT_MAX, True) + return f"{HINT_TITLE}\n{cleaned}\n\n" if cleaned else "" + + +def build_user_prompt(event: Event, history: Sequence[StateChange], hint: str = "") -> str: in_downtime = "ja" if event.suppress_reason else "nein" return ( + f"{_hint_section(hint)}" f"{DATA_OPEN}\n" f"Host: {sanitize(event.host, LABEL_MAX, True)}\n" f"Service: {sanitize(event.service, LABEL_MAX, True)}\n" @@ -63,8 +71,9 @@ def build_user_prompt(event: Event, history: Sequence[StateChange]) -> str: ) -def build_messages(editable: str | None, event: Event, history: Sequence[StateChange]) -> list[dict]: +def build_messages(editable: str | None, event: Event, history: Sequence[StateChange], + hint: str = "") -> list[dict]: return [ {"role": "system", "content": build_system_prompt(editable)}, - {"role": "user", "content": build_user_prompt(event, history)}, + {"role": "user", "content": build_user_prompt(event, history, hint)}, ] diff --git a/app/alerts/report_status.py b/app/alerts/report_status.py new file mode 100644 index 0000000..46cb383 --- /dev/null +++ b/app/alerts/report_status.py @@ -0,0 +1,35 @@ +import math +from dataclasses import dataclass +from typing import Mapping, Sequence + +from ..models import STATE_OK +from ..mutes import Mute, find_active_mute +from ..routing import Route, resolve_delay + +SECONDS_PER_MINUTE = 60 + +STATUS_REPORTED = "reported" +STATUS_WAITING = "waiting" +STATUS_SUPPRESSED = "suppressed" +STATUS_DUE = "due" + + +@dataclass(frozen=True) +class ReportStatus: + kind: str + label: str + + +def report_status(row: Mapping, rules: Sequence[Route], mutes: Sequence[Mute], default_minutes: int, + now: float) -> ReportStatus: + if row["in_downtime"]: + return ReportStatus(STATUS_SUPPRESSED, "unterdrückt (Downtime)") + if find_active_mute(mutes, row["host"], row["service"], now): + return ReportStatus(STATUS_SUPPRESSED, "unterdrückt (Stummschaltung)") + if row["flap_reported"] or (row["reported_state"] != STATE_OK and row["reported_state"] == row["state"]): + return ReportStatus(STATUS_REPORTED, "gemeldet") + delay = resolve_delay(rules, row["host"], row["service"], row["state"], row["output"], default_minutes) + remaining = delay * SECONDS_PER_MINUTE - (now - row["last_state_change"]) + if remaining <= 0: + return ReportStatus(STATUS_DUE, "wird beim nächsten Abruf gemeldet") + return ReportStatus(STATUS_WAITING, f"wartet noch {math.ceil(remaining / SECONDS_PER_MINUTE)} min") diff --git a/app/alerts/repository.py b/app/alerts/repository.py index f5eaa7a..f6115c9 100644 --- a/app/alerts/repository.py +++ b/app/alerts/repository.py @@ -1,6 +1,7 @@ from ..chat import MessageRef from ..db import Database -from ..models import AI_PENDING, AI_SKIPPED, STATE_OK, Event +from ..models import (AI_PENDING, AI_SKIPPED, KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, KIND_REMINDER, STATE_OK, + Event) RECENT_LIMIT = 200 DAY_SECONDS = 86400 @@ -45,6 +46,15 @@ class AlertRepository: rows = self._db.query("SELECT host, service FROM alerts WHERE id=?", (alert_id,)) return (rows[0]["host"], rows[0]["service"]) if rows else None + def report_counts(self, host: str, service: str) -> tuple[int, int]: + rows = self._db.query( + "SELECT COUNT(*) AS total, COALESCE(SUM(CASE WHEN suppressed=0 THEN 1 ELSE 0 END), 0) AS sent " + "FROM alerts WHERE host=? AND service=? AND kind IN (?,?,?) AND id > " + "COALESCE((SELECT MAX(id) FROM alerts WHERE host=? AND service=? AND kind=?), 0)", + (host, service, KIND_PROBLEM, KIND_FLAPPING, KIND_REMINDER, host, service, KIND_RECOVERY), + ) + return rows[0]["total"], rows[0]["sent"] + def recent(self, limit: int = RECENT_LIMIT) -> list[dict]: rows = self._db.query("SELECT * FROM alerts ORDER BY ts DESC, id DESC LIMIT ?", (limit,)) return [dict(row) for row in rows] @@ -58,7 +68,8 @@ class AlertRepository: def open_problems(self) -> list[dict]: rows = self._db.query( - "SELECT host, service, state, output, last_state_change, in_downtime, flapping " + "SELECT host, service, state, output, last_state_change, in_downtime, flapping, reported_state, " + "flap_reported " "FROM service_states WHERE state!=? ORDER BY state DESC, host, service", (STATE_OK,), ) return [dict(row) for row in rows] diff --git a/app/chat/base.py b/app/chat/base.py index e67cffc..0791706 100644 --- a/app/chat/base.py +++ b/app/chat/base.py @@ -52,9 +52,9 @@ class ChatAdapter(Protocol): name: str default_target: str - async def send(self, target: str, text: str, buttons: Buttons = ()) -> MessageRef: ... + async def send(self, target: str, text: str, buttons: Buttons = (), silent: bool = False) -> MessageRef: ... - async def edit(self, ref: MessageRef, text: str, buttons: Buttons = ()) -> None: ... + async def edit(self, ref: MessageRef, text: str, buttons: Buttons = (), silent: bool = False) -> None: ... async def check(self) -> str: ... diff --git a/app/chat/matrix.py b/app/chat/matrix.py index ed4faf5..0837de0 100644 --- a/app/chat/matrix.py +++ b/app/chat/matrix.py @@ -11,6 +11,7 @@ BACKEND_MATRIX = "matrix" API_PREFIX = "/_matrix/client/v3" HTML_FORMAT = "org.matrix.custom.html" MSGTYPE_TEXT = "m.text" +MSGTYPE_NOTICE = "m.notice" REL_REPLACE = "m.replace" EDIT_PREFIX = "* " DEFAULT_TIMEOUT = 15.0 @@ -28,9 +29,9 @@ def _enc(value: str) -> str: return quote(value, safe="") -def _content(text: str) -> dict: +def _content(text: str, silent: bool = False) -> dict: body, formatted = to_matrix(text) - return {"msgtype": MSGTYPE_TEXT, "body": body, "format": HTML_FORMAT, "formatted_body": formatted} + return {"msgtype": MSGTYPE_NOTICE if silent else MSGTYPE_TEXT, "body": body, "format": HTML_FORMAT, "formatted_body": formatted} class MatrixAdapter: @@ -78,13 +79,13 @@ class MatrixAdapter: raise MatrixError("Antwort ohne event_id") return str(event_id) - async def send(self, target: str, text: str, buttons: Buttons = ()) -> MessageRef: - return MessageRef(self.name, target, await self._put_message(target, _content(text))) + async def send(self, target: str, text: str, buttons: Buttons = (), silent: bool = False) -> MessageRef: + return MessageRef(self.name, target, await self._put_message(target, _content(text, silent))) - async def edit(self, ref: MessageRef, text: str, buttons: Buttons = ()) -> None: - new_content = _content(text) + async def edit(self, ref: MessageRef, text: str, buttons: Buttons = (), silent: bool = False) -> None: + new_content = _content(text, silent) content = { - "msgtype": MSGTYPE_TEXT, + "msgtype": new_content["msgtype"], "body": EDIT_PREFIX + new_content["body"], "format": HTML_FORMAT, "formatted_body": EDIT_PREFIX + new_content["formatted_body"], diff --git a/app/chat/telegram.py b/app/chat/telegram.py index 635f4ae..48dd497 100644 --- a/app/chat/telegram.py +++ b/app/chat/telegram.py @@ -59,8 +59,10 @@ class TelegramAdapter: raise TelegramError("Telegram nicht erreichbar (Rate-Limit)") async def send(self, target: str, text: str, buttons: Buttons = (), reply_to: str | None = None, - html: bool = True) -> MessageRef: + html: bool = True, silent: bool = False) -> MessageRef: payload: dict = {"chat_id": target, "text": text, "disable_web_page_preview": True} + if silent: + payload["disable_notification"] = True if html: payload["parse_mode"] = PARSE_MODE if buttons: @@ -70,7 +72,7 @@ class TelegramAdapter: result = await self._call("sendMessage", payload) return MessageRef(self.name, str(target), str(result["message_id"])) - async def edit(self, ref: MessageRef, text: str, buttons: Buttons = ()) -> None: + async def edit(self, ref: MessageRef, text: str, buttons: Buttons = (), silent: bool = False) -> None: payload: dict = {"chat_id": ref.target, "message_id": int(ref.message_id), "text": text, "parse_mode": PARSE_MODE, "disable_web_page_preview": True} if buttons: diff --git a/app/checkmk.py b/app/checkmk.py index 286ca09..9a87905 100644 --- a/app/checkmk.py +++ b/app/checkmk.py @@ -9,6 +9,10 @@ API_PATH = "/check_mk/api/1.0" SERVICES_PATH = "/domain-types/service/collections/all" VERSION_PATH = "/version" DOWNTIMES_PATH = "/domain-types/downtime/collections/all" +DOWNTIME_TYPE_PARAM = "downtime_type" +DOWNTIME_TYPE_HOST = "host" +DOWNTIME_TYPE_SERVICE = "service" +DOWNTIME_TYPES = (DOWNTIME_TYPE_HOST, DOWNTIME_TYPE_SERVICE) DEFAULT_TIMEOUT = 15.0 STATE_OK = 0 @@ -148,9 +152,14 @@ class CheckmkClient: return await self._services(_equals("host_name", host)) async def downtimes(self, host: str | None = None) -> list[Downtime]: - params = {"host_name": host} if host else None - body = await self._request("GET", DOWNTIMES_PATH, params=params) - return [_parse_downtime(item["extensions"]) for item in body.get("value", [])] + downtimes: list[Downtime] = [] + for downtime_type in DOWNTIME_TYPES: + body = await self._request("GET", DOWNTIMES_PATH, params={DOWNTIME_TYPE_PARAM: downtime_type}) + downtimes += [_parse_downtime(item["extensions"]) for item in body.get("value", [])] + if host: + wanted = host.casefold() + downtimes = [d for d in downtimes if d.host.casefold() == wanted] + return downtimes async def aclose(self) -> None: await self._client.aclose() diff --git a/app/constants.py b/app/constants.py index b5cb245..7a751d6 100644 --- a/app/constants.py +++ b/app/constants.py @@ -39,16 +39,21 @@ ADAPTER_SETTING_KEYS = ( ) POLL_INTERVAL = "poll.interval" -POLL_MIN_DURATION = "poll.min_duration" ALERTS_BUNDLE_SECONDS = "alerts.bundle_seconds" ALERTS_AI_PER_MINUTE = "alerts.ai_per_minute" +ALERTS_DEFAULT_DELAY_MINUTES = "alerts.default_delay_minutes" +ALERTS_RECOVERY_ENABLED = "alerts.recovery_enabled" +ALERTS_REMINDER_HOURS = "alerts.reminder_hours" PROMPT_ALERT_SYSTEM = "prompt.alert_system" DEFAULT_POLL_INTERVAL = 30 MIN_POLL_INTERVAL = 5 -DEFAULT_POLL_MIN_DURATION = 0 DEFAULT_BUNDLE_SECONDS = 60 DEFAULT_AI_PER_MINUTE = 10 +DEFAULT_DELAY_MINUTES = 15 +MAX_DELAY_MINUTES = 1440 +DEFAULT_REMINDER_HOURS = 0 +MAX_REMINDER_HOURS = 168 DEFAULT_TELEGRAM_API_BASE = "https://api.telegram.org" CHECKMK_DT_ENABLED = "checkmk_dt.enabled" diff --git a/app/db.py b/app/db.py index f02211c..1d54bf2 100644 --- a/app/db.py +++ b/app/db.py @@ -147,6 +147,25 @@ MIGRATIONS = [ ); CREATE INDEX state_log_ts ON state_log(ts); """, + """ + ALTER TABLE service_states ADD COLUMN problem_since INTEGER; + ALTER TABLE service_states ADD COLUMN reported_at INTEGER NOT NULL DEFAULT 0; + ALTER TABLE service_states ADD COLUMN reported_since INTEGER NOT NULL DEFAULT 0; + ALTER TABLE service_states ADD COLUMN last_reminder_ts INTEGER NOT NULL DEFAULT 0; + UPDATE service_states SET reported_at = CAST(strftime('%s', 'now') AS INTEGER), + reported_since = last_state_change WHERE reported_state != 0; + ALTER TABLE routes ADD COLUMN delay_minutes INTEGER; + ALTER TABLE routes ADD COLUMN output_contains TEXT NOT NULL DEFAULT ''; + ALTER TABLE routes ADD COLUMN ai_enabled INTEGER NOT NULL DEFAULT 1; + ALTER TABLE routes ADD COLUMN ai_hint TEXT NOT NULL DEFAULT ''; + ALTER TABLE routes ADD COLUMN reminder_hours INTEGER; + ALTER TABLE routes ADD COLUMN loud INTEGER NOT NULL DEFAULT 1; + ALTER TABLE routes ADD COLUMN use_default_targets INTEGER NOT NULL DEFAULT 0; + CREATE TABLE app_state ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + """, ] diff --git a/app/models.py b/app/models.py index d43906b..c8d0c68 100644 --- a/app/models.py +++ b/app/models.py @@ -3,12 +3,16 @@ from dataclasses import dataclass KIND_PROBLEM = "problem" KIND_RECOVERY = "recovery" KIND_FLAPPING = "flapping" +KIND_REMINDER = "reminder" STATE_OK = 0 +STATE_CRIT = 2 STATE_NAMES = {0: "OK", 1: "WARN", 2: "CRIT", 3: "UNKNOWN"} SUPPRESS_DOWNTIME = "downtime" SUPPRESS_MUTE = "mute" +SUPPRESS_RECOVERY_OFF = "recovery_aus" +SUPPRESS_NOT_REPORTED = "nicht_gemeldet" AI_PENDING = "pending" AI_DONE = "done" @@ -32,10 +36,16 @@ class Event: output: str = "" suppressed: bool = False suppress_reason: str | None = None + previous_state: int | None = None + previous_duration: int | None = None + since_ts: int | None = None + problem_since_ts: int | None = None + ai_enabled: bool = True + ai_hint: str = "" @property def wants_ai(self) -> bool: - return self.kind == KIND_PROBLEM and not self.suppressed + return self.kind == KIND_PROBLEM and not self.suppressed and self.ai_enabled @dataclass(frozen=True) @@ -49,6 +59,10 @@ class StateRow: flapping: bool reported_state: int flap_reported: bool + problem_since: int | None = None + reported_at: int = 0 + reported_since: int = 0 + last_reminder_ts: int = 0 @property def key(self) -> Key: diff --git a/app/poller/events.py b/app/poller/events.py index eb77a17..36d877c 100644 --- a/app/poller/events.py +++ b/app/poller/events.py @@ -1,38 +1,60 @@ -from typing import Mapping, Sequence +from dataclasses import replace +from typing import Callable, Mapping, Sequence from ..checkmk import ServiceState -from ..models import (KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, STATE_OK, SUPPRESS_DOWNTIME, Event, - Key, StateRow) +from ..models import (KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, STATE_OK, SUPPRESS_DOWNTIME, Event, Key, + StateRow) FLAP_QUIET_SECONDS = 600 - -def _event(now: int, kind: str, host: str, service: str, state: int, output: str, - in_downtime: bool) -> Event: - return Event(ts=now, host=host, service=service, kind=kind, state=state, output=output, - suppressed=in_downtime, suppress_reason=SUPPRESS_DOWNTIME if in_downtime else None) +DelayProvider = Callable[[str, str, int, str], int] -def _row(cur: ServiceState, reported_state: int, flap_reported: bool) -> StateRow: - return StateRow(cur.host, cur.service, cur.state, cur.output, cur.last_state_change, - cur.in_downtime, cur.flapping, reported_state, flap_reported) +def no_delay(host: str, service: str, state: int, output: str) -> int: + return 0 + + +def _problem_since(cur: ServiceState, prev: StateRow | None, now: int) -> int: + if prev is not None and prev.state != STATE_OK: + return prev.problem_since or prev.last_state_change + return cur.last_state_change or now + + +def _row(cur: ServiceState, prev: StateRow | None, now: int, reported_state: int, flap_reported: bool, + reported: tuple[int, int] | None = None) -> StateRow: + reported_at, reported_since = reported or ((prev.reported_at, prev.reported_since) if prev else (0, 0)) + return StateRow(cur.host, cur.service, cur.state, cur.output, cur.last_state_change, cur.in_downtime, + cur.flapping, reported_state, flap_reported, _problem_since(cur, prev, now), + reported_at, reported_since, prev.last_reminder_ts if prev else 0) + + +def _event(now: int, kind: str, cur: ServiceState, row: StateRow, **extra) -> Event: + return Event(ts=now, host=cur.host, service=cur.service, kind=kind, state=cur.state, output=cur.output, + suppressed=cur.in_downtime, suppress_reason=SUPPRESS_DOWNTIME if cur.in_downtime else None, + since_ts=cur.last_state_change, problem_since_ts=row.problem_since, **extra) + + +def _problem_event(now: int, cur: ServiceState, prev: StateRow | None, row: StateRow) -> Event: + if prev is None or prev.reported_state == STATE_OK: + return _event(now, KIND_PROBLEM, cur, row) + duration = max(cur.last_state_change - prev.reported_since, 0) if prev.reported_since else None + return _event(now, KIND_PROBLEM, cur, row, previous_state=prev.reported_state, previous_duration=duration) def _handle_current(cur: ServiceState, prev: StateRow | None, now: int, - min_duration: int) -> tuple[StateRow | None, Event | None]: + delay_seconds: DelayProvider) -> tuple[StateRow | None, Event | None]: if not cur.hard: return prev, None reported = prev.reported_state if prev else STATE_OK flap_reported = prev.flap_reported if prev else False if cur.flapping: - if flap_reported: - return _row(cur, reported, True), None - event = _event(now, KIND_FLAPPING, cur.host, cur.service, cur.state, cur.output, cur.in_downtime) - return _row(cur, reported, True), event - if now - cur.last_state_change < min_duration or cur.state == reported: - return _row(cur, reported, False), None - event = _event(now, KIND_PROBLEM, cur.host, cur.service, cur.state, cur.output, cur.in_downtime) - return _row(cur, cur.state, False), event + row = _row(cur, prev, now, reported, True) + return row, (None if flap_reported else _event(now, KIND_FLAPPING, cur, row)) + delay = delay_seconds(cur.host, cur.service, cur.state, cur.output) + if now - cur.last_state_change < delay or cur.state == reported: + return _row(cur, prev, now, reported, False), None + row = _row(cur, prev, now, cur.state, False, (now, cur.last_state_change)) + return row, _problem_event(now, cur, prev, row) def _handle_gone(prev: StateRow, now: int) -> tuple[StateRow | None, Event | None]: @@ -41,24 +63,27 @@ def _handle_gone(prev: StateRow, now: int) -> tuple[StateRow | None, Event | Non return None, None if prev.state == STATE_OK: return prev, None - held = StateRow(prev.host, prev.service, STATE_OK, "", now, False, True, - prev.reported_state, True) + held = replace(prev, state=STATE_OK, output="", last_state_change=now, in_downtime=False, + flapping=True, flap_reported=True) return held, None if prev.reported_state == STATE_OK: return None, None - event = _event(now, KIND_RECOVERY, prev.host, prev.service, STATE_OK, "", prev.in_downtime) + event = Event(ts=now, host=prev.host, service=prev.service, kind=KIND_RECOVERY, state=STATE_OK, + suppressed=prev.in_downtime, suppress_reason=SUPPRESS_DOWNTIME if prev.in_downtime else None, + previous_state=prev.reported_state, + problem_since_ts=prev.problem_since or prev.last_state_change) return None, event def compute_events(previous: Mapping[Key, StateRow], current: Sequence[ServiceState], now: int, - min_duration: int = 0) -> tuple[list[Event], list[StateRow]]: + delay_seconds: DelayProvider = no_delay) -> tuple[list[Event], list[StateRow]]: events: list[Event] = [] rows: dict[Key, StateRow] = {} seen: set[Key] = set() for cur in current: key = (cur.host, cur.service) seen.add(key) - row, event = _handle_current(cur, previous.get(key), now, min_duration) + row, event = _handle_current(cur, previous.get(key), now, delay_seconds) if row is not None: rows[key] = row if event is not None: diff --git a/app/poller/loop.py b/app/poller/loop.py index 2af76db..4ecfdcf 100644 --- a/app/poller/loop.py +++ b/app/poller/loop.py @@ -5,6 +5,7 @@ from typing import Awaitable, Callable from ..checkmk import CheckmkClient from ..models import Event +from ..routing import Route, delay_provider from .events import compute_events from .options import PollOptions, read_poll_options from .state import StateStore @@ -15,18 +16,20 @@ log = logging.getLogger(__name__) ERROR_MAX = 200 EventSink = Callable[[list[Event]], Awaitable[None]] +RuleSource = Callable[[], list[Route]] class Poller: def __init__(self, store, state: StateStore, sink: EventSink, status: PollerStatus, client_factory: Callable[[], CheckmkClient | None] | None = None, - clock: Callable[[], float] = time.time): + clock: Callable[[], float] = time.time, rules: RuleSource | None = None): self._store = store self._state = state self._sink = sink self.status = status self._client_factory = client_factory or (lambda: CheckmkClient.from_settings(store)) self._clock = clock + self._rules = rules or (lambda: []) async def run(self) -> None: while True: @@ -55,7 +58,9 @@ class Poller: await client.aclose() now = int(self._clock()) previous = await asyncio.to_thread(self._state.load) - events, rows = compute_events(previous, current, now, options.min_duration) + rules = await asyncio.to_thread(self._rules) + delay = delay_provider(rules, options.default_delay_minutes) + events, rows = compute_events(previous, current, now, delay) await asyncio.to_thread(self._state.apply, previous, rows, now) self.status.last_success = self._clock() self.status.last_error = None diff --git a/app/poller/options.py b/app/poller/options.py index fa22e6d..17db1e7 100644 --- a/app/poller/options.py +++ b/app/poller/options.py @@ -1,17 +1,21 @@ from dataclasses import dataclass -from ..constants import (DEFAULT_POLL_INTERVAL, DEFAULT_POLL_MIN_DURATION, MIN_POLL_INTERVAL, - POLL_INTERVAL, POLL_MIN_DURATION) +from ..constants import (ALERTS_DEFAULT_DELAY_MINUTES, DEFAULT_DELAY_MINUTES, DEFAULT_POLL_INTERVAL, + MAX_DELAY_MINUTES, MIN_POLL_INTERVAL, POLL_INTERVAL) @dataclass(frozen=True) class PollOptions: interval: int - min_duration: int + default_delay_minutes: int + + +def read_default_delay(store) -> int: + return min(store.get_int(ALERTS_DEFAULT_DELAY_MINUTES, DEFAULT_DELAY_MINUTES), MAX_DELAY_MINUTES) def read_poll_options(store) -> PollOptions: return PollOptions( interval=store.get_int(POLL_INTERVAL, DEFAULT_POLL_INTERVAL, MIN_POLL_INTERVAL), - min_duration=store.get_int(POLL_MIN_DURATION, DEFAULT_POLL_MIN_DURATION), + default_delay_minutes=read_default_delay(store), ) diff --git a/app/poller/state.py b/app/poller/state.py index 0aee5e2..e4ebb64 100644 --- a/app/poller/state.py +++ b/app/poller/state.py @@ -5,13 +5,16 @@ from ..models import STATE_OK, Key, StateRow RECENT_CHANGES_DEFAULT = 5 +# last_reminder_ts bleibt beim Update unangetastet, damit der Erinnerungs-Worker nicht überschrieben wird. UPSERT_SQL = ( "INSERT INTO service_states(host, service, state, output, last_state_change, in_downtime, " - "flapping, reported_state, flap_reported) VALUES (?,?,?,?,?,?,?,?,?) " + "flapping, reported_state, flap_reported, problem_since, reported_at, reported_since, " + "last_reminder_ts) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) " "ON CONFLICT(host, service) DO UPDATE SET state=excluded.state, output=excluded.output, " "last_state_change=excluded.last_state_change, in_downtime=excluded.in_downtime, " "flapping=excluded.flapping, reported_state=excluded.reported_state, " - "flap_reported=excluded.flap_reported" + "flap_reported=excluded.flap_reported, problem_since=excluded.problem_since, " + "reported_at=excluded.reported_at, reported_since=excluded.reported_since" ) LOG_SQL = "INSERT INTO state_log(host, service, state, ts) VALUES (?,?,?,?)" @@ -25,7 +28,8 @@ class StateChange: def _from_db(row) -> StateRow: return StateRow(row["host"], row["service"], row["state"], row["output"], row["last_state_change"], bool(row["in_downtime"]), bool(row["flapping"]), row["reported_state"], - bool(row["flap_reported"])) + bool(row["flap_reported"]), row["problem_since"], row["reported_at"], + row["reported_since"], row["last_reminder_ts"]) class StateStore: @@ -53,10 +57,15 @@ class StateStore: statements.append((LOG_SQL, (row.host, row.service, row.state, row.last_state_change or now))) statements.append((UPSERT_SQL, (row.host, row.service, row.state, row.output, row.last_state_change, int(row.in_downtime), int(row.flapping), - row.reported_state, int(row.flap_reported)))) + row.reported_state, int(row.flap_reported), row.problem_since, + row.reported_at, row.reported_since, row.last_reminder_ts))) if statements: self._db.execute_many(statements) + def mark_reminded(self, host: str, service: str, ts: int) -> None: + self._db.execute("UPDATE service_states SET last_reminder_ts=? WHERE host=? AND service=?", + (ts, host, service)) + def recent_changes(self, host: str, service: str, limit: int = RECENT_CHANGES_DEFAULT) -> list[StateChange]: rows = self._db.query( "SELECT ts, state FROM state_log WHERE host=? AND service=? ORDER BY ts DESC, id DESC LIMIT ?", diff --git a/app/reminders/__init__.py b/app/reminders/__init__.py new file mode 100644 index 0000000..ec8e178 --- /dev/null +++ b/app/reminders/__init__.py @@ -0,0 +1,4 @@ +from .due import due_reminders +from .worker import ReminderWorker + +__all__ = ["ReminderWorker", "due_reminders"] diff --git a/app/reminders/due.py b/app/reminders/due.py new file mode 100644 index 0000000..0812efa --- /dev/null +++ b/app/reminders/due.py @@ -0,0 +1,25 @@ +from typing import Iterable, Sequence + +from ..models import KIND_PROBLEM, KIND_REMINDER, STATE_CRIT, Event, StateRow +from ..mutes import Mute, find_active_mute +from ..routing import Route, reminder_interval + +SECONDS_PER_HOUR = 3600 + + +def due_reminders(rows: Iterable[StateRow], rules: Sequence[Route], mutes: Sequence[Mute], + default_hours: int, now: int) -> list[Event]: + events = [] + for row in rows: + if row.state != STATE_CRIT or row.reported_state != STATE_CRIT or row.in_downtime or row.flapping: + continue + if find_active_mute(mutes, row.host, row.service, now): + continue + probe = Event(now, row.host, row.service, KIND_PROBLEM, row.state, row.output) + hours = reminder_interval(rules, probe) or default_hours + if hours <= 0 or now - max(row.last_reminder_ts, row.reported_at) < hours * SECONDS_PER_HOUR: + continue + events.append(Event(now, row.host, row.service, KIND_REMINDER, row.state, row.output, + since_ts=row.last_state_change, problem_since_ts=row.problem_since)) + return events + diff --git a/app/reminders/worker.py b/app/reminders/worker.py new file mode 100644 index 0000000..4afb4ca --- /dev/null +++ b/app/reminders/worker.py @@ -0,0 +1,53 @@ +import asyncio +import logging +import time +from typing import Awaitable, Callable + +from ..models import Event, Key, StateRow +from ..mutes import Mute +from ..routing import Route +from .due import due_reminders + +log = logging.getLogger(__name__) + +CHECK_INTERVAL_SECONDS = 60.0 + + +class ReminderWorker: + def __init__(self, load_state: Callable[[], dict[Key, StateRow]], rules: Callable[[], list[Route]], + mutes: Callable[[float], list[Mute]], default_hours: Callable[[], int], + sink: Callable[[list[Event]], Awaitable[None]], mark: Callable[[str, str, int], None], + clock: Callable[[], float] = time.time, interval: float = CHECK_INTERVAL_SECONDS, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep): + self._load = load_state + self._rules = rules + self._mutes = mutes + self._default_hours = default_hours + self._sink = sink + self._mark = mark + self._clock = clock + self._interval = interval + self._sleep = sleep + + async def tick(self) -> int: + now = int(self._clock()) + rows = await asyncio.to_thread(self._load) + rules = await asyncio.to_thread(self._rules) + mutes = await asyncio.to_thread(self._mutes, now) + events = due_reminders(rows.values(), rules, mutes, self._default_hours(), now) + for event in events: + # Zeitpunkt zuerst festhalten: lieber eine Erinnerung zu wenig als eine doppelte nach Neustart. + await asyncio.to_thread(self._mark, event.host, event.service, now) + if events: + await self._sink(events) + return len(events) + + async def run(self) -> None: + while True: + try: + await self.tick() + except asyncio.CancelledError: + raise + except Exception: + log.exception("Erinnerungen: Prüfung fehlgeschlagen") + await self._sleep(self._interval) diff --git a/app/routing/__init__.py b/app/routing/__init__.py index 1289f67..ff3083c 100644 --- a/app/routing/__init__.py +++ b/app/routing/__init__.py @@ -1,9 +1,14 @@ +from .decisions import ai_allowed, ai_hints, delay_provider, reminder_interval, resolve_delay, resolve_loud +from .limits import DEFAULT_MIN_STATE from .repository import RouteRepository -from .rules import ALL_KINDS, DEFAULT_MIN_STATE, Route, resolve_targets, route_matches +from .rules import ALL_KINDS, Route, matches, matching_rules, resolve_targets, route_matches +from .seed import ZPOOL_MARKER, seed_default_rules, zpool_rule from .service import DIRECTION_DOWN, DIRECTION_UP, RouteError, RouteService from .validation import BACKENDS, RouteInput, validate_route __all__ = [ "ALL_KINDS", "BACKENDS", "DEFAULT_MIN_STATE", "DIRECTION_DOWN", "DIRECTION_UP", "Route", "RouteError", - "RouteInput", "RouteRepository", "RouteService", "resolve_targets", "route_matches", "validate_route", + "RouteInput", "RouteRepository", "RouteService", "ZPOOL_MARKER", "ai_allowed", "ai_hints", + "delay_provider", "matches", "matching_rules", "reminder_interval", "resolve_delay", "resolve_loud", + "resolve_targets", "route_matches", "seed_default_rules", "validate_route", "zpool_rule", ] diff --git a/app/routing/decisions.py b/app/routing/decisions.py new file mode 100644 index 0000000..943aafb --- /dev/null +++ b/app/routing/decisions.py @@ -0,0 +1,43 @@ +from typing import Callable, Sequence + +from ..models import KIND_PROBLEM, Event +from .limits import AI_HINTS_TOTAL_MAX +from .rules import Route, matching_rules + +SECONDS_PER_MINUTE = 60 + + +def _problem_probe(host: str, service: str, state: int, output: str) -> Event: + return Event(0, host, service, KIND_PROBLEM, state, output) + + +def resolve_delay(rules: Sequence[Route], host: str, service: str, state: int, output: str, + default_minutes: int) -> int: + found = matching_rules(rules, _problem_probe(host, service, state, output)) + delays = [r.delay_minutes for r in found if r.delay_minutes is not None] + return min(delays) if delays else default_minutes + + +def delay_provider(rules: Sequence[Route], default_minutes: int) -> Callable[[str, str, int, str], int]: + def seconds(host: str, service: str, state: int, output: str) -> int: + return resolve_delay(rules, host, service, state, output, default_minutes) * SECONDS_PER_MINUTE + return seconds + + +def ai_allowed(rules: Sequence[Route], event: Event) -> bool: + return all(rule.ai_enabled for rule in matching_rules(rules, event)) + + +def resolve_loud(rules: Sequence[Route], event: Event) -> bool: + found = matching_rules(rules, event) + return not found or any(rule.loud for rule in found) + + +def ai_hints(rules: Sequence[Route], event: Event) -> str: + hints = [rule.ai_hint.strip() for rule in matching_rules(rules, event) if rule.ai_hint.strip()] + return "\n".join(hints)[:AI_HINTS_TOTAL_MAX] + + +def reminder_interval(rules: Sequence[Route], event: Event) -> int: + hours = [r.reminder_hours for r in matching_rules(rules, event) if r.reminder_hours and r.reminder_hours > 0] + return min(hours) if hours else 0 diff --git a/app/routing/limits.py b/app/routing/limits.py new file mode 100644 index 0000000..408ffed --- /dev/null +++ b/app/routing/limits.py @@ -0,0 +1,9 @@ +NAME_MAX = 80 +TARGET_MAX = 200 +OUTPUT_CONTAINS_MAX = 200 +AI_HINT_MAX = 800 +AI_HINTS_TOTAL_MAX = 1200 +DELAY_MINUTES_RANGE = (0, 1440) +REMINDER_HOURS_RANGE = (0, 168) +MIN_STATE_RANGE = (0, 3) +DEFAULT_MIN_STATE = 1 diff --git a/app/routing/repository.py b/app/routing/repository.py index f1f9a99..e067d1f 100644 --- a/app/routing/repository.py +++ b/app/routing/repository.py @@ -6,21 +6,41 @@ from .rules import Route from .validation import RouteInput COLUMNS = ("id, name, position, enabled, match_host, match_service, min_state, kinds, stop_on_match, " - "targets") + "targets, delay_minutes, output_contains, ai_enabled, ai_hint, reminder_hours, loud, " + "use_default_targets") KINDS_SEP = "," +INSERT_SQL = ( + "INSERT INTO routes(name, enabled, match_host, match_service, min_state, kinds, stop_on_match, " + "targets, delay_minutes, output_contains, ai_enabled, ai_hint, reminder_hours, loud, " + "use_default_targets, position) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, " + "(SELECT COALESCE(MAX(position), 0) + 1 FROM routes))" +) +UPDATE_SQL = ( + "UPDATE routes SET name=?, enabled=?, match_host=?, match_service=?, min_state=?, kinds=?, " + "stop_on_match=?, targets=?, delay_minutes=?, output_contains=?, ai_enabled=?, ai_hint=?, " + "reminder_hours=?, loud=?, use_default_targets=? WHERE id=?" +) def _route(row) -> Route: targets = tuple(Target(t["backend"], t["target"]) for t in json.loads(row["targets"])) kinds = tuple(k for k in row["kinds"].split(KINDS_SEP) if k) return Route(row["id"], row["name"], row["position"], bool(row["enabled"]), row["match_host"], - row["match_service"], row["min_state"], kinds, bool(row["stop_on_match"]), targets) + row["match_service"], row["min_state"], kinds, bool(row["stop_on_match"]), targets, + row["delay_minutes"], row["output_contains"], bool(row["ai_enabled"]), row["ai_hint"], + row["reminder_hours"], bool(row["loud"]), bool(row["use_default_targets"])) def _values(data: RouteInput) -> tuple: targets = json.dumps([t._asdict() for t in data.targets]) return (data.name.strip(), int(data.enabled), data.match_host.strip(), data.match_service.strip(), - data.min_state, KINDS_SEP.join(data.kinds), int(data.stop_on_match), targets) + data.min_state, KINDS_SEP.join(data.kinds), int(data.stop_on_match), targets, + data.delay_minutes, data.output_contains.strip(), int(data.ai_enabled), data.ai_hint.strip(), + data.reminder_hours or None, int(data.loud), int(data.use_default_targets)) + + +def insert_statement(data: RouteInput) -> tuple[str, tuple]: + return INSERT_SQL, _values(data) class RouteRepository: @@ -36,17 +56,10 @@ class RouteRepository: return _route(rows[0]) if rows else None def insert(self, data: RouteInput) -> int: - return self._db.insert( - "INSERT INTO routes(name, enabled, match_host, match_service, min_state, kinds, stop_on_match, " - "targets, position) VALUES (?,?,?,?,?,?,?,?, (SELECT COALESCE(MAX(position), 0) + 1 FROM routes))", - _values(data), - ) + return self._db.insert(*insert_statement(data)) def update(self, route_id: int, data: RouteInput) -> bool: - return self._db.execute( - "UPDATE routes SET name=?, enabled=?, match_host=?, match_service=?, min_state=?, kinds=?, " - "stop_on_match=?, targets=? WHERE id=?", (*_values(data), route_id), - ) > 0 + return self._db.execute(UPDATE_SQL, (*_values(data), route_id)) > 0 def set_enabled(self, route_id: int, enabled: bool) -> bool: return self._db.execute("UPDATE routes SET enabled=? WHERE id=?", (int(enabled), route_id)) > 0 diff --git a/app/routing/rules.py b/app/routing/rules.py index d6545d4..7114c62 100644 --- a/app/routing/rules.py +++ b/app/routing/rules.py @@ -3,11 +3,13 @@ from typing import Sequence from ..chat.base import Target from ..globs import glob_matches -from ..models import KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, Event +from ..models import KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, KIND_REMINDER, Event +from .limits import DEFAULT_MIN_STATE, MIN_STATE_RANGE ALL_KINDS = (KIND_PROBLEM, KIND_RECOVERY, KIND_FLAPPING) -DEFAULT_MIN_STATE = 1 -MIN_STATE_RANGE = (0, 3) + +__all__ = ["ALL_KINDS", "DEFAULT_MIN_STATE", "MIN_STATE_RANGE", "Route", "matches", "matching_rules", + "resolve_targets", "route_matches"] @dataclass(frozen=True) @@ -22,26 +24,48 @@ class Route: kinds: tuple[str, ...] stop_on_match: bool targets: tuple[Target, ...] + delay_minutes: int | None = None + output_contains: str = "" + ai_enabled: bool = True + ai_hint: str = "" + reminder_hours: int | None = None + loud: bool = True + use_default_targets: bool = False + + +def matches(route: Route, host: str, service: str, kind: str, state: int, output: str) -> bool: + # Erinnerungen folgen den Regeln für Probleme, damit keine eigene Alarmart nötig ist. + kind = KIND_PROBLEM if kind == KIND_REMINDER else kind + if not route.enabled or kind not in route.kinds: + return False + if kind == KIND_PROBLEM and state < route.min_state: + return False + if route.output_contains and route.output_contains.lower() not in output.lower(): + return False + return glob_matches(route.match_host, host) and glob_matches(route.match_service, service) def route_matches(route: Route, event: Event) -> bool: - if not route.enabled or event.kind not in route.kinds: - return False - if event.kind == KIND_PROBLEM and event.state < route.min_state: - return False - return glob_matches(route.match_host, event.host) and glob_matches(route.match_service, event.service) + return matches(route, event.host, event.service, event.kind, event.state, event.output) -def resolve_targets(rules: Sequence[Route], event: Event, defaults: Sequence[Target]) -> list[Target]: - resolved: list[Target] = [] - matched = False +def matching_rules(rules: Sequence[Route], event: Event) -> list[Route]: + found: list[Route] = [] for rule in sorted(rules, key=lambda r: (r.position, r.id)): if not route_matches(rule, event): continue - matched = True - resolved.extend(t for t in rule.targets if t not in resolved) + found.append(rule) if rule.stop_on_match: break - if not matched: + return found + + +def resolve_targets(rules: Sequence[Route], event: Event, defaults: Sequence[Target]) -> list[Target]: + found = matching_rules(rules, event) + if not found: return list(dict.fromkeys(defaults)) + resolved: list[Target] = [] + for rule in found: + extra = defaults if rule.use_default_targets else () + resolved.extend(t for t in (*rule.targets, *extra) if t not in resolved) return resolved diff --git a/app/routing/seed.py b/app/routing/seed.py new file mode 100644 index 0000000..cb2dc40 --- /dev/null +++ b/app/routing/seed.py @@ -0,0 +1,41 @@ +import logging + +from .. import audit +from ..db import Database +from ..models import KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY +from .repository import insert_statement +from .validation import RouteInput + +log = logging.getLogger(__name__) + +SEED_ACTOR = "system" +ZPOOL_MARKER = "seed.rule.zfs_zpool" +ZPOOL_REMINDER_HOURS = 4 +ZPOOL_HINT = ( + "ZFS-Pool-Alarm. Typische Prüfschritte: zpool status -v (betroffenes Gerät, Fehlerzähler " + "READ/WRITE/CKSUM), läuft ein Scrub oder Resilver, SMART-Werte der betroffenen Platte, " + "Kabel/Controller, ob 'zpool clear' genügt oder ein 'zpool replace' nötig ist. Der Meldungstext " + "enthält oft keine Geräte- oder Fehlerzahlen; weise darauf hin, dass sie nicht verifiziert sind." +) + + +def zpool_rule() -> RouteInput: + return RouteInput( + name="ZFS Zpool", match_host="*", match_service="*zpool*", min_state=1, + kinds=(KIND_PROBLEM, KIND_RECOVERY, KIND_FLAPPING), enabled=True, use_default_targets=True, + delay_minutes=0, ai_enabled=True, reminder_hours=ZPOOL_REMINDER_HOURS, loud=True, + ai_hint=ZPOOL_HINT, + ) + + +def seed_default_rules(db: Database) -> bool: + try: + if db.query("SELECT 1 FROM app_state WHERE key=?", (ZPOOL_MARKER,)): + return False + sql, params = insert_statement(zpool_rule()) + db.execute_many([(sql, params), ("INSERT INTO app_state(key, value) VALUES (?, '1')", (ZPOOL_MARKER,))]) + audit.record(db, SEED_ACTOR, "route.seed", zpool_rule().name) + return True + except Exception: + log.exception("Standardregel konnte nicht angelegt werden") + return False diff --git a/app/routing/service.py b/app/routing/service.py index 71c0622..ce48ad4 100644 --- a/app/routing/service.py +++ b/app/routing/service.py @@ -17,9 +17,13 @@ class RouteError(ValueError): def describe(route: Route | RouteInput) -> str: targets = ", ".join(f"{t.backend}:{t.target}" for t in route.targets) or "keine" state = "an" if route.enabled else "aus" + delay = "std" if route.delay_minutes is None else route.delay_minutes return (f"{route.name} [{state}] host={route.match_host or '*'} service={route.match_service or '*'} " f"min_state={route.min_state} kinds={','.join(route.kinds)} " - f"stop={int(route.stop_on_match)} -> {targets}") + f"stop={int(route.stop_on_match)} delay={delay} contains={route.output_contains!r} " + f"ai={int(route.ai_enabled)} hint_len={len(route.ai_hint)} " + f"reminder_h={route.reminder_hours or 0} loud={int(route.loud)} " + f"default_targets={int(route.use_default_targets)} -> {targets}") class RouteService: diff --git a/app/routing/validation.py b/app/routing/validation.py index 72af741..c242369 100644 --- a/app/routing/validation.py +++ b/app/routing/validation.py @@ -1,13 +1,15 @@ +import re from dataclasses import dataclass, field from ..chat.base import Target from ..chat.targets import valid_target from ..globs import validate_glob -from .rules import ALL_KINDS, DEFAULT_MIN_STATE, MIN_STATE_RANGE +from .limits import (AI_HINT_MAX, DEFAULT_MIN_STATE, DELAY_MINUTES_RANGE, MIN_STATE_RANGE, NAME_MAX, + OUTPUT_CONTAINS_MAX, REMINDER_HOURS_RANGE, TARGET_MAX) +from .rules import ALL_KINDS -NAME_MAX = 80 -TARGET_MAX = 200 BACKENDS = ("telegram", "matrix") +CONTROL_RE = re.compile(r"[\x00-\x08\x0b-\x1f\x7f]") @dataclass(frozen=True) @@ -20,6 +22,21 @@ class RouteInput: stop_on_match: bool = False enabled: bool = True targets: tuple[Target, ...] = field(default_factory=tuple) + delay_minutes: int | None = None + output_contains: str = "" + ai_enabled: bool = True + ai_hint: str = "" + reminder_hours: int | None = None + loud: bool = True + use_default_targets: bool = False + + +def _range_error(value: int | None, bounds: tuple[int, int], label: str, unit: str = "") -> list[str]: + low, high = bounds + if value is not None and not low <= value <= high: + suffix = f" {unit}" if unit else "" + return [f"{label} muss zwischen {low} und {high}{suffix} liegen."] + return [] def validate_route(data: RouteInput) -> list[str]: @@ -29,11 +46,19 @@ def validate_route(data: RouteInput) -> list[str]: elif len(data.name) > NAME_MAX: errors.append(f"Name ist zu lang (max. {NAME_MAX} Zeichen).") errors += validate_glob(data.match_host, "Host-Muster") + validate_glob(data.match_service, "Service-Muster") - low, high = MIN_STATE_RANGE - if not low <= data.min_state <= high: - errors.append(f"Mindest-Schweregrad muss zwischen {low} und {high} liegen.") + errors += _range_error(data.min_state, MIN_STATE_RANGE, "Mindest-Schweregrad") if not data.kinds or any(kind not in ALL_KINDS for kind in data.kinds): errors.append("Mindestens eine gültige Alarmart wählen.") + errors += _range_error(data.delay_minutes, DELAY_MINUTES_RANGE, "Verzögerung", "Minuten") + errors += _range_error(data.reminder_hours, REMINDER_HOURS_RANGE, "Erinnerung", "Stunden") + if len(data.output_contains) > OUTPUT_CONTAINS_MAX: + errors.append(f"Der Ausgabe-Text ist zu lang (max. {OUTPUT_CONTAINS_MAX} Zeichen).") + if CONTROL_RE.search(data.output_contains): + errors.append("Der Ausgabe-Text enthält Steuerzeichen.") + if len(data.ai_hint) > AI_HINT_MAX: + errors.append(f"Der KI-Hinweis ist zu lang (max. {AI_HINT_MAX} Zeichen).") + if CONTROL_RE.search(data.ai_hint): + errors.append("Der KI-Hinweis enthält Steuerzeichen.") for target in data.targets: if target.backend not in BACKENDS: errors.append(f"Unbekanntes Backend: {target.backend}") diff --git a/app/runtime.py b/app/runtime.py index 523dbf3..a28d285 100644 --- a/app/runtime.py +++ b/app/runtime.py @@ -8,7 +8,8 @@ from . import audit from .alerts import AlertActions, AlertAnalyzer, AlertPipeline, AlertRepository, SlidingWindowLimiter from .chat import AdapterRegistry from .chatops import ChatOps -from .constants import ALERTS_BUNDLE_SECONDS, DEFAULT_BUNDLE_SECONDS +from .constants import (ALERTS_BUNDLE_SECONDS, ALERTS_RECOVERY_ENABLED, ALERTS_REMINDER_HOURS, DEFAULT_BUNDLE_SECONDS, + DEFAULT_REMINDER_HOURS, FLAG_OFF, FLAG_ON, MAX_REMINDER_HOURS) from .db import Database from .agent_runs import configured_secrets from .mutes import MuteService @@ -17,11 +18,13 @@ from .poller import Poller, PollerStatus, StateStore from .prompts import (PROMPT_AGENT, PROMPT_ALERT, PromptRepository, PromptService, PromptTester, UserRateLimiter, default_registry) from .retention import RetentionRepository, RetentionRunner, RetentionService, policy_from -from .routing import RouteService +from .reminders import ReminderWorker +from .routing import RouteService, seed_default_rules class Runtime: def __init__(self, db: Database, store): + self._db = db self.status = PollerStatus() self.alerts = AlertRepository(db) self.state = StateStore(db) @@ -39,8 +42,15 @@ class Runtime: self.alerts, self.registry.current, analyzer, lambda: store.get_int(ALERTS_BUNDLE_SECONDS, DEFAULT_BUNDLE_SECONDS), routes=self.routes, mutes=self.mutes, actions=AlertActions(store), + recovery_enabled=lambda: store.get(ALERTS_RECOVERY_ENABLED, FLAG_ON) != FLAG_OFF, + ) + self.poller = Poller(store, self.state, self.pipeline.handle, self.status, + rules=self.routes.enabled_routes) + self.reminders = ReminderWorker( + self.state.load, self.routes.enabled_routes, self.mutes.active, + lambda: min(store.get_int(ALERTS_REMINDER_HOURS, DEFAULT_REMINDER_HOURS), MAX_REMINDER_HOURS), + self.pipeline.handle, self.state.mark_reminded, ) - self.poller = Poller(store, self.state, self.pipeline.handle, self.status) self.chatops = ChatOps(db, store, self.state, self.alerts, self.mutes, lambda: self.prompts.effective_text(PROMPT_AGENT)) self.prompt_tester = PromptTester( @@ -49,7 +59,9 @@ class Runtime: @asynccontextmanager async def lifespan(self, app: FastAPI): + await asyncio.to_thread(seed_default_rules, self._db) poller_task = asyncio.create_task(self.poller.run()) + reminder_task = asyncio.create_task(self.reminders.run()) pipeline_task = asyncio.create_task(self.pipeline.run()) listener_tasks = [asyncio.create_task(run()) for run in self.chatops.listener_runners()] retention_task = asyncio.create_task(self.retention_runner.run()) @@ -62,7 +74,8 @@ class Runtime: retention_task.cancel() await asyncio.gather(retention_task, return_exceptions=True) poller_task.cancel() - await asyncio.gather(poller_task, return_exceptions=True) + reminder_task.cancel() + await asyncio.gather(poller_task, reminder_task, return_exceptions=True) await self.pipeline.finish() pipeline_task.cancel() await asyncio.gather(pipeline_task, return_exceptions=True) diff --git a/app/web/alerting_section.py b/app/web/alerting_section.py index b4aa3bf..006b14d 100644 --- a/app/web/alerting_section.py +++ b/app/web/alerting_section.py @@ -1,18 +1,20 @@ -from ..constants import (ALERTS_AI_PER_MINUTE, ALERTS_BUNDLE_SECONDS, DEFAULT_AI_PER_MINUTE, - DEFAULT_BUNDLE_SECONDS, DEFAULT_POLL_INTERVAL, DEFAULT_POLL_MIN_DURATION, - MIN_POLL_INTERVAL, POLL_INTERVAL, POLL_MIN_DURATION) +from ..constants import (ALERTS_AI_PER_MINUTE, ALERTS_BUNDLE_SECONDS, ALERTS_DEFAULT_DELAY_MINUTES, + ALERTS_RECOVERY_ENABLED, ALERTS_REMINDER_HOURS, DEFAULT_AI_PER_MINUTE, + DEFAULT_BUNDLE_SECONDS, DEFAULT_DELAY_MINUTES, DEFAULT_POLL_INTERVAL, + DEFAULT_REMINDER_HOURS, MAX_DELAY_MINUTES, MAX_REMINDER_HOURS, MIN_POLL_INTERVAL, + POLL_INTERVAL) from .sections import Field, Section SLUG = "alerting" MAX_POLL_INTERVAL = 3600 -MAX_MIN_DURATION = 3600 MAX_BUNDLE_SECONDS = 3600 MAX_AI_PER_MINUTE = 600 LIMITS = ( (POLL_INTERVAL, "Abfrageintervall", MIN_POLL_INTERVAL, MAX_POLL_INTERVAL), - (POLL_MIN_DURATION, "Mindestdauer", 0, MAX_MIN_DURATION), + (ALERTS_DEFAULT_DELAY_MINUTES, "Standard-Verzögerung", 0, MAX_DELAY_MINUTES), + (ALERTS_REMINDER_HOURS, "Erinnerung", 0, MAX_REMINDER_HOURS), (ALERTS_BUNDLE_SECONDS, "Bündelungsfenster", 0, MAX_BUNDLE_SECONDS), (ALERTS_AI_PER_MINUTE, "KI-Analysen pro Minute", 0, MAX_AI_PER_MINUTE), ) @@ -34,8 +36,13 @@ SECTION = Section( title="Alarmierung", fields=( Field(POLL_INTERVAL, "Abfrageintervall in Sekunden", default=str(DEFAULT_POLL_INTERVAL)), - Field(POLL_MIN_DURATION, "Mindestdauer eines Problems in Sekunden (0 = sofort)", - default=str(DEFAULT_POLL_MIN_DURATION)), + Field(ALERTS_DEFAULT_DELAY_MINUTES, + "Standard-Verzögerung in Minuten (Problem wird erst nach dieser Zeit gemeldet, 0 = sofort; " + "Regeln können das überschreiben)", default=str(DEFAULT_DELAY_MINUTES)), + Field(ALERTS_RECOVERY_ENABLED, "Recovery-Meldungen senden (wieder OK)", kind="checkbox"), + Field(ALERTS_REMINDER_HOURS, + "Standard-Erinnerung für offene CRIT-Probleme in Stunden (0 = aus; Regeln überschreiben)", + default=str(DEFAULT_REMINDER_HOURS)), Field(ALERTS_BUNDLE_SECONDS, "Bündelungsfenster pro Host in Sekunden", default=str(DEFAULT_BUNDLE_SECONDS)), Field(ALERTS_AI_PER_MINUTE, "Max. KI-Analysen pro Minute (0 = keine)", diff --git a/app/web/alerts.py b/app/web/alerts.py index 0d128f1..92e607b 100644 --- a/app/web/alerts.py +++ b/app/web/alerts.py @@ -5,10 +5,12 @@ from .deps import current_user, render router = APIRouter() +SUPPRESS_LABELS = {"mute": "Stummschaltung", "downtime": "Downtime", "recovery_aus": "Recovery aus", + "nicht_gemeldet": "Problem war nicht gemeldet"} AI_LABELS = {"pending": "läuft", "done": "fertig", "failed": "nicht verfügbar", "skipped": "übersprungen"} @router.get("/alerts") def alerts_page(request: Request, user: User = Depends(current_user)): entries = request.app.state.runtime.alerts.recent() - return render(request, "alerts.html", user=user, entries=entries, ai_labels=AI_LABELS) + return render(request, "alerts.html", user=user, entries=entries, ai_labels=AI_LABELS, suppress_labels=SUPPRESS_LABELS) diff --git a/app/web/pages.py b/app/web/pages.py index 38a2d93..7e10f60 100644 --- a/app/web/pages.py +++ b/app/web/pages.py @@ -16,6 +16,7 @@ from ..ollama import OllamaClient from ..users import MIN_PASSWORD_LEN, ROLES, User, UserError from . import alerting_section, downtime_section, matrix_section, retention_section, telegram_section from .deps import (FLASH_ERROR, FLASH_OK, current_user, flash, redirect, render, require_admin) +from .problem_status import problems_with_status from .sections import (URL_RE, Field, Section, check, collect, effective, find_section, form_values, secret_flags, section_done, stored_values) @@ -135,7 +136,7 @@ async def dashboard(request: Request, user: User = Depends(current_user)): section_done(store, s) for s in SECTIONS.values() if not s.optional) return render(request, "dashboard.html", user=user, result=result, setup_open=setup_open, alerts_24h=runtime.alerts.count_since(int(time.time()) - DAY_SECONDS), - open_problems=runtime.alerts.open_problems()) + open_problems=problems_with_status(request)) @router.get("/setup") diff --git a/app/web/problem_status.py b/app/web/problem_status.py new file mode 100644 index 0000000..4f979be --- /dev/null +++ b/app/web/problem_status.py @@ -0,0 +1,16 @@ +import time + +from fastapi import Request + +from ..alerts import report_status +from ..poller.options import read_default_delay + + +def problems_with_status(request: Request) -> list[dict]: + runtime = request.app.state.runtime + now = time.time() + rules = runtime.routes.enabled_routes() + mutes = runtime.mutes.active(now) + default_minutes = read_default_delay(request.app.state.store) + return [{**row, "report": report_status(row, rules, mutes, default_minutes, now)} + for row in runtime.alerts.open_problems()] diff --git a/app/web/routing_form.py b/app/web/routing_form.py new file mode 100644 index 0000000..c4f8b0f --- /dev/null +++ b/app/web/routing_form.py @@ -0,0 +1,78 @@ +from fastapi import Request + +from ..chat import Target, default_targets +from ..models import KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, STATE_NAMES +from ..routing import DEFAULT_MIN_STATE, Route, RouteInput + +CHOICE_SEP = "|" +EXTRA_SEP = ":" +KIND_LABELS = {KIND_PROBLEM: "Problem", KIND_RECOVERY: "Recovery", KIND_FLAPPING: "Flapping"} + + +def _choice_value(target: Target) -> str: + return f"{target.backend}{CHOICE_SEP}{target.target}" + + +def _optional_int(form, name: str, label: str, errors: list[str]) -> int | None: + raw = str(form.get(name, "")).strip() + if not raw: + return None + try: + return int(raw) + except ValueError: + errors.append(f"{label} muss eine ganze Zahl sein.") + return None + + +def _parse_targets(form, errors: list[str]) -> tuple[Target, ...]: + targets: list[Target] = [] + for value in form.getlist("targets"): + backend, _, target = str(value).partition(CHOICE_SEP) + targets.append(Target(backend, target)) + for line in str(form.get("extra_targets", "")).splitlines(): + if not line.strip(): + continue + backend, sep, target = line.strip().partition(EXTRA_SEP) + if not sep: + errors.append(f"Zielzeile ungültig: {line.strip()[:60]} (erwartet backend:ziel).") + continue + targets.append(Target(backend.strip().lower(), target.strip())) + return tuple(dict.fromkeys(targets)) + + +def parse_form(form) -> tuple[RouteInput, list[str]]: + errors: list[str] = [] + min_state = _optional_int(form, "min_state", "Mindest-Schweregrad", errors) + data = RouteInput( + name=str(form.get("name", "")).strip(), match_host=str(form.get("match_host", "")).strip(), + match_service=str(form.get("match_service", "")).strip(), + min_state=DEFAULT_MIN_STATE if min_state is None else min_state, + kinds=tuple(str(k) for k in form.getlist("kinds")), stop_on_match=bool(form.get("stop_on_match")), + enabled=bool(form.get("enabled")), targets=_parse_targets(form, errors), + delay_minutes=_optional_int(form, "delay_minutes", "Verzögerung", errors), + output_contains=str(form.get("output_contains", "")).strip(), + ai_enabled=bool(form.get("ai_enabled")), ai_hint=str(form.get("ai_hint", "")).strip(), + reminder_hours=_optional_int(form, "reminder_hours", "Erinnerung", errors), + loud=bool(form.get("loud")), use_default_targets=bool(form.get("use_default_targets")), + ) + return data, errors + + +def form_context(request: Request, data: RouteInput) -> dict: + choices = default_targets(request.app.state.store) + free = [t for t in data.targets if t not in choices] + return { + "form": data, + "choices": [{"value": _choice_value(t), "label": f"{t.backend}: {t.target}"} for t in choices], + "selected": {_choice_value(t) for t in data.targets if t in choices}, + "extra_text": "\n".join(f"{t.backend}{EXTRA_SEP}{t.target}" for t in free), + "state_options": sorted(STATE_NAMES.items()), + "kind_options": KIND_LABELS, + } + + +def to_input(route: Route) -> RouteInput: + return RouteInput(route.name, route.match_host, route.match_service, route.min_state, route.kinds, + route.stop_on_match, route.enabled, route.targets, route.delay_minutes, + route.output_contains, route.ai_enabled, route.ai_hint, route.reminder_hours, + route.loud, route.use_default_targets) diff --git a/app/web/routing_pages.py b/app/web/routing_pages.py index 88dc686..b6aacca 100644 --- a/app/web/routing_pages.py +++ b/app/web/routing_pages.py @@ -1,20 +1,16 @@ from fastapi import APIRouter, Depends, Form, HTTPException, Request -from ..chat import Target, default_targets from ..constants import FLAG_ON -from ..models import KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, STATE_NAMES -from ..routing import (DEFAULT_MIN_STATE, DIRECTION_DOWN, DIRECTION_UP, Route, RouteError, RouteInput, - RouteService) +from ..models import STATE_NAMES +from ..routing import DIRECTION_DOWN, DIRECTION_UP, Route, RouteError, RouteInput, RouteService from ..users import User from .deps import FLASH_ERROR, FLASH_OK, current_user, flash, redirect, render, require_admin +from .routing_form import KIND_LABELS, form_context, parse_form, to_input router = APIRouter() PATH = "/routing" STATUS_INVALID = 422 -CHOICE_SEP = "|" -EXTRA_SEP = ":" -KIND_LABELS = {KIND_PROBLEM: "Problem", KIND_RECOVERY: "Recovery", KIND_FLAPPING: "Flapping"} MSG_NOT_FOUND = "Regel nicht gefunden." @@ -22,60 +18,10 @@ def _service(request: Request) -> RouteService: return request.app.state.runtime.routes -def _choice_value(target: Target) -> str: - return f"{target.backend}{CHOICE_SEP}{target.target}" - - -def parse_form(form) -> tuple[RouteInput, list[str]]: - errors: list[str] = [] - try: - min_state = int(str(form.get("min_state", DEFAULT_MIN_STATE)).strip()) - except ValueError: - min_state = DEFAULT_MIN_STATE - errors.append("Mindest-Schweregrad muss eine Zahl sein.") - targets: list[Target] = [] - for value in form.getlist("targets"): - backend, _, target = str(value).partition(CHOICE_SEP) - targets.append(Target(backend, target)) - for line in str(form.get("extra_targets", "")).splitlines(): - if not line.strip(): - continue - backend, sep, target = line.strip().partition(EXTRA_SEP) - if not sep: - errors.append(f"Zielzeile ungültig: {line.strip()[:60]} (erwartet backend:ziel).") - continue - targets.append(Target(backend.strip().lower(), target.strip())) - data = RouteInput( - name=str(form.get("name", "")).strip(), match_host=str(form.get("match_host", "")).strip(), - match_service=str(form.get("match_service", "")).strip(), min_state=min_state, - kinds=tuple(str(k) for k in form.getlist("kinds")), stop_on_match=bool(form.get("stop_on_match")), - enabled=bool(form.get("enabled")), targets=tuple(dict.fromkeys(targets)), - ) - return data, errors - - -def _form_context(request: Request, data: RouteInput) -> dict: - choices = default_targets(request.app.state.store) - free = [t for t in data.targets if t not in choices] - return { - "form": data, - "choices": [{"value": _choice_value(t), "label": f"{t.backend}: {t.target}"} for t in choices], - "selected": {_choice_value(t) for t in data.targets if t in choices}, - "extra_text": "\n".join(f"{t.backend}{EXTRA_SEP}{t.target}" for t in free), - "state_options": sorted(STATE_NAMES.items()), - "kind_options": KIND_LABELS, - } - - -def _to_input(route: Route) -> RouteInput: - return RouteInput(route.name, route.match_host, route.match_service, route.min_state, route.kinds, - route.stop_on_match, route.enabled, route.targets) - - def _list_page(request: Request, user: User, data: RouteInput, status_code: int = 200): return render(request, "routing.html", user=user, routes=_service(request).list_all(), state_names=STATE_NAMES, kind_labels=KIND_LABELS, status_code=status_code, - **_form_context(request, data)) + **form_context(request, data)) def _get_or_404(request: Request, route_id: int) -> Route: @@ -109,7 +55,7 @@ async def routing_create(request: Request, admin: User = Depends(require_admin)) def routing_edit_page(request: Request, route_id: int, admin: User = Depends(require_admin)): route = _get_or_404(request, route_id) return render(request, "route_edit.html", user=admin, route=route, - **_form_context(request, _to_input(route))) + **form_context(request, to_input(route))) @router.post(PATH + "/{route_id}") @@ -124,7 +70,7 @@ async def routing_update(request: Request, route_id: int, admin: User = Depends( for message in exc.errors: flash(request, message, FLASH_ERROR) return render(request, "route_edit.html", user=admin, route=route, status_code=STATUS_INVALID, - **_form_context(request, data)) + **form_context(request, data)) flash(request, "Regel gespeichert.", FLASH_OK) return redirect(PATH) diff --git a/app/web/templates/_route_form.html b/app/web/templates/_route_form.html index 6f976a6..059a4ce 100644 --- a/app/web/templates/_route_form.html +++ b/app/web/templates/_route_form.html @@ -10,6 +10,8 @@ + +
Alarmarten {% for kind, label in kind_options.items() %} @@ -23,9 +25,21 @@ {% else %}

Kein Chat-Backend mit Standardziel aktiviert.

{% endfor %} + -

Ohne Ziele wird bei Treffer nichts gesendet.

+

Ohne Ziele und ohne Standardziele wird bei Treffer nichts gesendet.

+
+
+ Verhalten + + + + + + + +
diff --git a/app/web/templates/alerts.html b/app/web/templates/alerts.html index dc666bf..5c1f541 100644 --- a/app/web/templates/alerts.html +++ b/app/web/templates/alerts.html @@ -14,7 +14,7 @@ {{ a.service }} {{ a.kind }} {{ a.state | state_name }} - {{ a.output }}{% if a.suppressed %} unterdrückt ({{ 'Stummschaltung' if a.suppress_reason == 'mute' else 'Downtime' }}){% endif %} + {{ a.output }}{% if a.suppressed %} unterdrückt ({{ suppress_labels.get(a.suppress_reason, a.suppress_reason) }}){% endif %} {{ ai_labels.get(a.ai_status, a.ai_status) }} {% if a.ai_text %}
{{ a.ai_text }}
{% endif %} diff --git a/app/web/templates/base.html b/app/web/templates/base.html index 2a4df1e..aef9b56 100644 --- a/app/web/templates/base.html +++ b/app/web/templates/base.html @@ -16,7 +16,7 @@