From 347f55cf7c305837acc1e6d379b33cfe7215ea14 Mon Sep 17 00:00:00 2001 From: patrick Date: Fri, 6 Nov 2026 03:06:23 +0100 Subject: [PATCH] feat: Phase 6c baseline, ack, host alarms, outage notice, send limit, privacy mode, UI link Includes atomic migrations, per-rule reminder semantics, first-run helper script and the note on the undecided UI chat idea. Co-Authored-By: Claude Sonnet 5.5 --- CLAUDE.md | 18 ++- README.md | 21 ++- app/alerts/__init__.py | 8 +- app/alerts/content.py | 31 ++++ app/alerts/delta.py | 22 ++- app/alerts/extras.py | 31 +++- app/alerts/followup.py | 46 ++++++ app/alerts/formatter.py | 46 ++++-- app/alerts/gate.py | 5 +- app/alerts/pipeline.py | 106 ++++++++----- app/alerts/prompts.py | 4 +- app/alerts/report_status.py | 14 +- app/alerts/repository.py | 9 +- app/alerts/sendlimit.py | 38 +++++ app/alerts/system.py | 53 +++++++ app/alerts/system_text.py | 87 +++++++++++ app/chat/base.py | 3 +- app/chat/telegram.py | 33 +++- app/checkmk.py | 91 ++++++++++- app/constants.py | 13 ++ app/db.py | 185 ++-------------------- app/main.py | 5 +- app/migrate.py | 46 ++++++ app/migrations.py | 174 +++++++++++++++++++++ app/models.py | 29 ++++ app/poller/__init__.py | 10 +- app/poller/baseline.py | 38 +++++ app/poller/events.py | 69 ++++++--- app/poller/loop.py | 62 +++++++- app/poller/notify.py | 12 ++ app/poller/options.py | 11 +- app/poller/outage.py | 47 ++++++ app/poller/state.py | 15 +- app/reminders/due.py | 8 +- app/routing/__init__.py | 4 +- app/routing/decisions.py | 15 +- app/routing/repository.py | 2 +- app/routing/service.py | 3 +- app/runtime.py | 16 +- app/web/alerting_section.py | 39 ++++- app/web/alerts.py | 3 +- app/web/baseline_pages.py | 22 +++ app/web/content_mode.py | 19 +++ app/web/deps.py | 3 +- app/web/matrix_section.py | 10 +- app/web/problem_status.py | 12 +- app/web/sections.py | 5 + app/web/telegram_section.py | 9 +- app/web/templates/_route_form.html | 2 +- app/web/templates/alerts.html | 4 +- app/web/templates/dashboard.html | 2 +- app/web/templates/routing.html | 2 +- app/web/templates/settings.html | 6 + deploy/README.md | 18 +-- deploy/first-run.sh | 114 ++++++++++++++ tests/test_checkmk_hosts.py | 110 +++++++++++++ tests/test_db_migrations.py | 101 ++++++++++++ tests/test_formatter_host_short.py | 95 ++++++++++++ tests/test_import_all.py | 27 ++++ tests/test_pipeline.py | 4 +- tests/test_pipeline_phase6c.py | 165 ++++++++++++++++++++ tests/test_pipeline_routing.py | 3 + tests/test_poller.py | 3 + tests/test_poller_ack_host.py | 193 +++++++++++++++++++++++ tests/test_poller_system.py | 240 +++++++++++++++++++++++++++++ tests/test_reminders.py | 13 +- tests/test_rule_decisions.py | 21 ++- tests/test_system_notifier.py | 140 +++++++++++++++++ tests/test_templates_compile.py | 14 ++ tests/test_ui_link.py | 187 ++++++++++++++++++++++ tests/test_web_phase6c.py | 174 +++++++++++++++++++++ tests/test_web_rules.py | 17 ++ 72 files changed, 2838 insertions(+), 369 deletions(-) create mode 100644 app/alerts/content.py create mode 100644 app/alerts/followup.py create mode 100644 app/alerts/sendlimit.py create mode 100644 app/alerts/system.py create mode 100644 app/alerts/system_text.py create mode 100644 app/migrate.py create mode 100644 app/migrations.py create mode 100644 app/poller/baseline.py create mode 100644 app/poller/notify.py create mode 100644 app/poller/outage.py create mode 100644 app/web/baseline_pages.py create mode 100644 app/web/content_mode.py create mode 100755 deploy/first-run.sh create mode 100644 tests/test_checkmk_hosts.py create mode 100644 tests/test_db_migrations.py create mode 100644 tests/test_formatter_host_short.py create mode 100644 tests/test_import_all.py create mode 100644 tests/test_pipeline_phase6c.py create mode 100644 tests/test_poller_ack_host.py create mode 100644 tests/test_poller_system.py create mode 100644 tests/test_system_notifier.py create mode 100644 tests/test_templates_compile.py create mode 100644 tests/test_ui_link.py create mode 100644 tests/test_web_phase6c.py diff --git a/CLAUDE.md b/CLAUDE.md index 91e81db..4d2e749 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ Vollständiger Plan: `/home/sysops/.claude/plans/ja-alle-bernehmen-eager-lark.md ## Arbeitsregeln - Auf diesem Rechner wird nichts installiert (kein venv, kein pip, keine Pakete). Code nur schreiben, Ausführung und Tests passieren im Ziel-LXC. -- Stand: Phase 1 bis 6 und 6b als Code, Phase 7 als Dateien geschrieben, nichts ausgeführt oder getestet (siehe README-Statustabelle). +- Stand: Phase 1 bis 6, 6b und 6c als Code, Phase 7 als Dateien geschrieben, nichts ausgeführt oder getestet (siehe README-Statustabelle). - Code-Kommentare nur bei echtem WHY. ## Festgelegte Entscheidungen @@ -75,13 +75,22 @@ tests/ - Offene Annahmen: Ollama-Retries im Testbereich wie im Betrieb (Request kann bis zu 3 x Timeout dauern); der Setup-Assistent zeigt Aufbewahrung wie Alarmierung als "erledigt", weil kein Pflichtfeld existiert. ## Phase 6b Regeln, Verzögerung, Delta, Erinnerungen (Code geschrieben, ungetestet) -- Migration Index 6: `service_states` + `problem_since`, `reported_at` (Zeitpunkt der letzten Meldung, bei Altzeilen = Migrationszeit), `reported_since` (Beginn des gemeldeten Zustands), `last_reminder_ts` (wird vom Poller-Upsert nie überschrieben); `routes` + `delay_minutes` (NULL = Standard), `output_contains`, `ai_enabled`, `ai_hint`, `reminder_hours` (NULL/0 = keine), `loud`, `use_default_targets`; neue Tabelle `app_state` (key/value, Marker `seed.rule.zfs_zpool`). Bis Index 6 belegt, 6c nimmt Index 7. +- Migration Index 6: `service_states` + `problem_since`, `reported_at` (Zeitpunkt der letzten Meldung, bei Altzeilen = Migrationszeit), `reported_since` (Beginn des gemeldeten Zustands), `last_reminder_ts` (wird vom Poller-Upsert nie überschrieben); `routes` + `delay_minutes` (NULL = Standard), `output_contains`, `ai_enabled`, `ai_hint`, `reminder_hours` (6b: NULL/0 = keine, ab 6c NULL = Standard, 0 = aus), `loud`, `use_default_targets`; neue Tabelle `app_state` (key/value, Marker `seed.rule.zfs_zpool`). Bis Index 6 belegt, 6c nimmt Index 7. - Settings: `alerts.default_delay_minutes` (15, 0-1440), `alerts.recovery_enabled` (an), `alerts.reminder_hours` (0, 0-168). `poll.min_duration` ist entfernt, der Altwert in `settings` bleibt unbeachtet liegen. - `app/routing/`: `rules.py` (Route, `matches`, `matching_rules` mit stop_on_match, `resolve_targets` inkl. use_default_targets), `decisions.py` (`resolve_delay`, `delay_provider`, `ai_allowed`, `resolve_loud`, `ai_hints` max. 1200, `reminder_interval`), `seed.py` (Zpool-Regel einmalig, Fehler nur geloggt, Aufruf im Lifespan), `limits.py`. Erinnerungen matchen wie Probleme (Art `reminder` nutzt `problem` der Regel). - Poller: `compute_events(previous, current, now, delay_seconds)` mit Callable pro (Host, Service, Status, Output); Regeln pro Zyklus frisch (`Poller(..., rules=routes.enabled_routes)`). `Event` hat `previous_state`, `previous_duration`, `since_ts`, `problem_since_ts`, `ai_enabled`, `ai_hint`; Art `reminder`. - Pipeline: `alerts/gate.py` (`ReportGate`: merkt pro Service, ob das Problem gesendet wurde, Fallback aus `alerts` per `report_counts`; Recovery-Schalter und "Recovery nur nach gemeldetem Problem", Gründe `recovery_aus`, `nicht_gemeldet`), Gruppenschlüssel (Host, Ziele, leise); Adapter `send/edit(..., silent=False)`, `Deliverer` übergibt `silent` nur wenn gesetzt. `alerts/delta.py` (Dauerformat, Delta-Texte), `alerts/report_status.py` (Dashboard). - `app/reminders/`: `due_reminders` (rein) und `ReminderWorker` (60 s, Zeitpunkt wird vor dem Senden gespeichert, im Lifespan gestartet). +## Phase 6c Betriebsrobustheit (Code geschrieben, ungetestet) +- Migrationen: Index 7 (`service_states` + `ack_state` INTEGER NULL = nicht bestätigt, sonst Status bei Beginn der Bestätigung; + `host_held` = Service-Alarm wegen Host down zurückgehalten, noch nicht als gemeldet gezählt; setzt Marker `baseline.done` in `app_state`, wenn `service_states` schon Zeilen hat), Index 8 (`routes.reminder_hours` 0 wird NULL). Die Liste liegt jetzt in `app/migrations.py`, der Runner (`app/migrate.py`: `split_statements`, `MigrationError`, je Migration `BEGIN IMMEDIATE` ... `PRAGMA user_version` ... `COMMIT`, sonst `ROLLBACK`) wird von `Database(path, migrations=None)` benutzt; der Autocommit-Modus gilt nur während der Migration. +- Erinnerung je Regel: `resolve_reminder_hours` (leer = Standard, 0 = aus, mehrere Treffer: eine 0 gewinnt, sonst kleinster Wert > 0, sonst Standard). Bestätigte Services bekommen keine Erinnerung. +- Poller: `Poller(..., notifier, baseline, tracker)`. Host-Zustände kommen als `ServiceState` mit Service `Host` (`HostState.as_service`) durch `compute_events`; Host-Verzögerung `alerts.host_delay_minutes` über `delay_provider(..., host_minutes)`. `build_row`/`baseline_rows` in `poller/events.py`/`poller/baseline.py`; `BaselineGuard` (Marker `baseline.done`, Anforderung `baseline.requested` in `app_state`), `OutageTracker` (Schwelle 3, `unreachable` = Transport/Timeout/5xx, `unexpected` = 4xx/Parse), `CheckmkError.kind/.brief` (brief nie mit Antwortinhalt). Ein Fehler beim Host-Abruf zählt als Fehlzyklus des ganzen Abrufs (sonst entstünden falsche Host-Recoveries). +- Pipeline/Alerts: `alerts/sendlimit.py` (`SendLimiter`, pro `Target`, Sammelmeldung per Ticker `_flush_overflow`), `alerts/content.py` (Modus-/Limit-/Basis-URL-Provider, `alert_link`), `alerts/system.py` + `system_text.py` (`SystemNotifier` für Baseline und Ausfall, Art `system`), `alerts/followup.py` (KI-Edit), Gruppenschlüssel `(host, targets, silent, modes)`. `ReportGate.reset` nach Baseline. Neue Suppress-Gründe `bestaetigt`, `host_down`, `sendelimit`. +- Settings: `alerts.host_delay_minutes` (0, 0-1440), `alerts.send_limit_per_minute` (15, 1-60), `ui.base_url`, `telegram.content_mode`, `matrix.content_mode` (`voll`/`kurz`). UI: Feldtyp `select`, `Field.hint`/`transform`, Aktion `POST /settings/alerting/baseline` (Admin, CSRF, Audit `baseline.request`), Telegram-Button `Details` per `Button.url` (Fallback ohne Link-Button bei abgelehnter URL). +- Annahmen: Host-Abruf `POST /domain-types/host/collections/all` mit Query `state != 0` und Spalten `name`, `state`, `plugin_output`, `last_state_change`, `scheduled_downtime_depth`, `acknowledged`; Service-Spalte `acknowledged`; Host-State 0/1/2 = UP/DOWN/UNREACH (intern 0/2/3); Hosts gelten immer als hart. Alles ungeprüft gegen 2.5 (Phase 6d prüft). +- Abweichung: Datenschutzmodus gilt je Backend, nicht je einzelnem Ziel (Ziele stehen in Regeln als freie Zeilen). Ein vom Sendelimit verworfenes Problem zählt im Speicher des Gates als gemeldet (die Recovery kommt). + ## Weiterer Weg (entschieden) Reihenfolge: Testbasis, erster Lauf im Test-LXC (`pytest`), Korrekturrunde, Phase 6, Phase 6b, Phase 6c, Phase 8, Abnahme, Parallelbetrieb, Tag `v0.1.0`. Verfeinerungen passieren am Live-System, nicht vorab. - Testbasis: `pytest` manuell im Test-LXC (Dev-venv mit `requirements-dev.txt`), dazu `.gitea/workflows/test.yml` für einen späteren Gitea-Actions-Runner. Logging als Zeilenlog nach journald, Level über `CHECKBOT_LOG_LEVEL` (Default INFO), nie Secrets in Logs (Test für die Adapter). Zeitzone über `CHECKBOT_TZ` im Setup-Skript, Default `Europe/Berlin`. @@ -104,8 +113,11 @@ Reihenfolge: Testbasis, erster Lauf im Test-LXC (`pytest`), Korrekturrunde, Phas - Delta in der Alarmnachricht (Phase 6b): `Event` bekommt Vorzustand und Dauer. Alarmzeile zeigt Übergang und Dauer (`WARN → CRIT, seit 18:05 (vorher WARN seit 2 h)`), Recovery die Gesamtdauer (`wieder OK nach 3 h 12 min (zuletzt CRIT)`), Erinnerungen `weiter CRIT seit 4 h`. - Recovery-Meldungen sind in der UI schaltbar: globaler Schalter `alerts.recovery_enabled` (Default an) in der Section "Alarmierung" und pro Regel über die Ereignisarten (`kinds` enthält `recovery` oder nicht). Greift zusätzlich zur Regel "Recovery nur, wenn das Problem vorher gemeldet wurde". Ist Recovery aus, bleibt sie im Verlauf sichtbar, wird aber nicht gesendet. - Verzögerung (Phase 6b): Section "Alarmierung" bekommt `alerts.default_delay_minutes` (Werkseinstellung 15; ersetzt `poll.min_duration`). Jede Regel kann die Verzögerung überschreiben, auch mit 0. Passen mehrere Regeln, gilt die kleinste. Sie zählt ab der letzten Statusänderung, eine Verschlechterung startet sie neu; wird das Problem vorher OK, gibt es weder Alarm noch Recovery. Das Dashboard zeigt wartende Probleme ("wartet noch 8 min"). Mute und Downtime wirken auch während der Wartezeit. Host-Alarme und der Checkmk-Ausfall (Phase 6c) kommen immer sofort, Host-Verzögerung `alerts.host_delay_minutes` Standard 0. Die vorinstallierte Regel "ZFS Zpool" (`*zpool*`) hat Verzögerung 0, damit gilt ab Werk: Zpool sofort, alles andere nach 15 min. +- Phase 6d (nach 6c): API-Selbsttest. Lesend beim Start, täglich 04:00 und sofort bei Checkmk-Versionswechsel: `/version`, Service- und Host-Abfrage (alle benötigten Spalten in der Antwort), Downtime-Abruf (GET). Ergebnis auf einer UI-Seite, Teilcheck `checkmk_api` in `/health`, Chat-Meldung bei Fehlschlag oder Versionswechsel (und bei Rückkehr zu ok). Unterscheidet "nicht erreichbar" von "Antwort unerwartet". +- Entschieden nach Phase 6b: Erinnerung pro Regel: leer = Standard, 0 = aus, >0 = Stunden. Keine Recovery ohne vorherigen Alarm bleibt so. Migrationen laufen künftig in einer Transaktion pro Migration mit Rollback. `deploy/first-run.sh` richtet das Dev-venv ein, lässt `pytest` laufen und packt Log, Python-Version, Paketliste und Git-Stand ohne Secrets in ein Archiv. Zugriff auf den Test-LXC per SSH nur dort, eigener Benutzer ohne Root, jeder Befehl mit Freigabe. Nach dem ersten grünen `pytest` und vor der Abnahme: `docs/THREAT_MODEL.md` (STRIDE) und ein einmaliger Sicherheits-Review des gesamten Codes. +- Vorgemerkt, noch NICHT entschieden (Phase 10, Idee): Chat mit dem Agenten ("Hermes") in der UI. Heute ist der Agent nur per Telegram/Matrix (`/ask`, `/explain`, Tiefenanalyse) und als Einzelfrage-Testlauf unter `/prompts` erreichbar. Vorschläge zur späteren Entscheidung: nur Admins (Schalter `ui.chat_viewers_allowed`, Standard aus); 6 Runden Kontext, Gespräche mit Gesprächs-ID in `agent_runs`, Nutzer sehen eigene Gespräche, Admins alle unter `/agent`, Aufbewahrung 90 Tage, Rate-Limit 5 pro Minute; Antwort nach Abschluss ohne Streaming ("läuft …" mit Abbrechen, max. 90 s), Tool-Schritte aufklappbar. Das Agentenmodell ist in der Section "Ollama" einstellbar (auch ein Hermes-Modell). Erst nach der Abnahme angehen. - Aktionen: Der Bot führt nie Shell-Befehle aus (weder lokal noch per SSH). Webhook pro Regel (POST JSON an Allowlist-URLs, Audit) kommt erst nach der Abnahme. - Phase 8: Integrationstests mit Marker `integration` (`pytest -m integration`), Zugangsdaten per Umgebung, nur lesend; Downtime nur mit ausdrücklichem Flag und gegen einen Testhost. Testhost `checkbot-test` in Checkmk mit schaltbarem lokalem Check plus halbautomatisches `tests/e2e/run.py`. - Abnahme (`docs/ACCEPTANCE.md`): CRIT in Telegram und Element, KI-Analyse binnen 90 s, Recovery ohne KI, Downtime und `mute` unterdrücken, Ollama aus ergibt Rohalarm mit "KI nicht verfügbar", Neustart ohne Doppelalarme, Backup und Restore real getestet, `/health` im Checkmk grün, Baseline-Zusammenfassung beim Erstlauf, Host-down unterdrückt Service-Alarme, Meldung bei Checkmk-Ausfall, 24 h Dauerlauf ohne Fehler im Journal. - Einführung: erste Woche nur Testziele (separate Gruppe/Raum) per Routing-Regel, danach auf Produktivziele umschalten. Release: Tag `v0.1.0` und kurze `CHANGELOG.md`. -- Bewusst nicht enthalten: Ack, mehrere Checkmk-Sites, weitere Kanäle (Mail/Slack), E2EE in Matrix. +- Bewusst nicht enthalten: Ack setzen (Bestätigungen aus Checkmk werden nur gelesen, Phase 6c), mehrere Checkmk-Sites, weitere Kanäle (Mail/Slack), E2EE in Matrix. diff --git a/README.md b/README.md index d6fda35..5ccf99e 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Bot für Checkmk. Er liest den Zustand per REST-API, lässt Probleme von einem e ### Verzögerung, Delta, Erinnerungen und Recovery - Verzögerung: Ein Problem wird erst gemeldet, wenn sein jetziger Zustand seit der Standard-Verzögerung (Alarmierung, Werkseinstellung 15 min, 0 = sofort) besteht; eine Regel kann das überschreiben, auch mit 0. Die Zeit zählt ab der letzten Statusänderung in Checkmk, eine Verschlechterung startet sie neu. Wird das Problem vorher wieder OK, gibt es weder Alarm noch Recovery. Mute und Downtime wirken auch in der Wartezeit; was bei Ablauf der Verzögerung unterdrückt ist, wird nicht nachgeholt. Die frühere Einstellung `poll.min_duration` gibt es nicht mehr, ein vorhandener Altwert wird ignoriert. - Delta in der Nachricht: `WARN → CRIT, seit 18:05 (vorher WARN seit 2 h)`, bei Recovery `wieder OK nach 3 h 12 min (zuletzt CRIT)`, bei Erinnerungen `weiter CRIT seit 4 h: `. Zeiten in der lokalen Zeit des Prozesses. -- Erinnerungen: nur für offene CRIT-Probleme, die gemeldet wurden und weiter CRIT sind, nicht bei Mute, Downtime oder Flapping. Intervall aus den passenden Regeln (kleinstes), sonst die Standard-Erinnerung in Stunden (Alarmierung, Werkseinstellung 0 = aus). Gemessen ab der letzten Meldung bzw. Erinnerung, der Zeitpunkt ist in der Datenbank gespeichert (kein Doppelversand nach Neustart). Erinnerungen laufen ohne KI durch dasselbe Routing wie Alarme und stehen im Verlauf als Art `reminder`. +- Erinnerungen: nur für offene CRIT-Probleme, die gemeldet wurden und weiter CRIT sind, nicht bei Mute, Downtime oder Flapping. Intervall aus den passenden Regeln (leer = Standard, 0 = aus für diese Regel, sonst der kleinste Wert), sonst die Standard-Erinnerung in Stunden (Alarmierung, Werkseinstellung 0 = aus). Bestätigte Probleme bekommen keine Erinnerung. Gemessen ab der letzten Meldung bzw. Erinnerung, der Zeitpunkt ist in der Datenbank gespeichert (kein Doppelversand nach Neustart). Erinnerungen laufen ohne KI durch dasselbe Routing wie Alarme und stehen im Verlauf als Art `reminder`. - Recovery: Schalter "Recovery-Meldungen senden" (Alarmierung, Werkseinstellung an) und pro Regel über die Alarmarten. Eine Recovery wird nur gesendet, wenn das Problem vorher tatsächlich gemeldet wurde; sonst (oder bei ausgeschaltetem Schalter) steht sie unterdrückt im Verlauf. ### KI-Analyse @@ -29,7 +29,7 @@ Bot für Checkmk. Er liest den Zustand per REST-API, lässt Probleme von einem e - Beide Backends parallel, einzeln abschaltbar. - Matrix läuft direkt über die Client-Server-API (`httpx`, kein `matrix-nio`, keine E2EE-Abhängigkeiten). Der Alarmraum muss unverschlüsselt sein, der Bot-User braucht einen Access-Token und muss dem Raum beigetreten sein. Die Analyse kommt per `m.replace`-Edit. - Regeln (UI `/routing`, in der Navigation "Regeln"): Host-/Service-Glob, optional "Ausgabe enthält" (Teilstring, keine Regex), Mindest-Schweregrad, Alarmarten und Ziele (Backend plus Chat-ID bzw. Raum-ID). Regeln gelten von oben nach unten, jede passende fügt ihre Ziele hinzu, optional mit Stopp nach Treffer. "Standardziele" in einer Regel fügt die Standardziele aller aktivierten Backends zu den expliziten Zielen hinzu. Eine passende Regel ganz ohne Ziele und ohne Standardziele sendet bewusst nichts. Passt keine Regel, gehen Alarme an die Standardziele. Alarme eines Hosts mit gleicher Zielmenge und gleicher Lautstärke werden zu einer Nachricht gebündelt. -- Weitere Regeloptionen: Verzögerung in Minuten (leer = Standard), KI an/aus, KI-Hinweis (max. 800 Zeichen), Erinnerung bei CRIT in Stunden (leer oder 0 = keine) und laut/leise (leise: Telegram ohne Benachrichtigung, Matrix als `m.notice`). Mehrere passende Regeln: die kleinste Verzögerung und das kleinste Erinnerungsintervall gelten, KI ist aus, sobald eine passende Regel sie ausschaltet, laut gilt, sobald eine passende Regel laut ist, KI-Hinweise werden in Regelreihenfolge zusammengefügt (insgesamt max. 1200 Zeichen). +- Weitere Regeloptionen: Verzögerung in Minuten (leer = Standard), KI an/aus, KI-Hinweis (max. 800 Zeichen), Erinnerung bei CRIT in Stunden (leer = Standard aus "Alarmierung", 0 = aus für diese Regel) und laut/leise (leise: Telegram ohne Benachrichtigung, Matrix als `m.notice`). Mehrere passende Regeln: die kleinste Verzögerung gilt, beim Erinnerungsintervall schaltet eine ausdrückliche 0 die Erinnerung aus, sonst gilt der kleinste Wert größer 0 (ohne Wert in allen Regeln der Standard), KI ist aus, sobald eine passende Regel sie ausschaltet, laut gilt, sobald eine passende Regel laut ist, KI-Hinweise werden in Regelreihenfolge zusammengefügt (insgesamt max. 1200 Zeichen). - Vorinstallierte Regel "ZFS Zpool" (einmalig beim ersten Start angelegt, danach editier- und löschbar, eine gelöschte Regel kommt nicht zurück): Service `*zpool*` ab WARN, Standardziele, Verzögerung 0, KI mit Hinweis auf typische Prüfschritte, Erinnerung bei CRIT alle 4 h. - Stummschaltung (UI `/mutes`): Host-/Service-Glob mit Ablauf (30 min bis 24 h oder freie Minuten) und Grund. Passende Probleme, Flapping und Recoveries werden nicht gemeldet und im Verlauf als unterdrückt (Stummschaltung) geführt. Abgelaufene Stummschaltungen räumt der Bot selbst auf. - Der Versand läuft über eine Warteschlange (max. 1000 Events, bei Überlauf wird das älteste verworfen), ein langsames Chat-Backend bremst den Poller nicht. @@ -77,9 +77,18 @@ Bot für Checkmk. Er liest den Zustand per REST-API, lässt Probleme von einem e - Der Bot räumt einmal nach dem Start (kurz verzögert) und danach einmal pro Kalendertag nach 03:00 Uhr lokaler Zeit auf, in Batches zu 1000 Zeilen mit Pausen. Aktuelle Service-Zustände und der Listener-Zustand bleiben unberührt. Nach dem Lauf folgt `PRAGMA wal_checkpoint(TRUNCATE)`, ein `VACUUM` gibt es bewusst nicht. - Die gelöschten Zeilenzahlen stehen im Journal (INFO) und als ein Audit-Eintrag `retention.run`. Die Einstellungsseite zeigt den letzten Lauf mit Zählern und hat den Button "Jetzt aufräumen" (nur Admins). +### Betriebsrobustheit (Phase 6c) +- Baseline beim Erstlauf: Ist der Zustandsspeicher leer und die Baseline noch nicht gesetzt, zeichnet der erste erfolgreiche Abruf alle offenen Probleme still auf (sie gelten als gemeldet, damit spätere Recoveries ankommen) und sendet eine einzige Zusammenfassung ohne KI an die Standardziele: Anzahl je Status inklusive Host-Problemen und die 10 kritischsten (Hosts zuerst, dann CRIT, UNKNOWN, WARN). Plugin-Output steht nie darin. Der Button "Baseline neu setzen" (Alarmierung, nur Admins, mit Audit-Eintrag) fordert dasselbe für den nächsten erfolgreichen Abruf an. +- Bestätigte Probleme (Acknowledged in Checkmk): kein neues Problem-Alarm, im Verlauf "unterdrückt (bestätigt)". Verschlechtert sich der Status gegenüber dem Stand bei Beginn der Bestätigung (Reihenfolge CRIT vor UNKNOWN vor WARN), gilt das Problem wieder als unbestätigt und wird gemeldet. Wird die Bestätigung aufgehoben, wird nichts nachgeholt. Stummschaltung, Downtime und Verzögerung wirken wie bisher; Recovery kommt weiter nur, wenn das Problem gemeldet wurde. +- Host-Alarme: Hosts, die nicht UP sind, werden mitabgefragt und als eigener Alarm gemeldet ("Host DOWN", "Host UNREACH", Recovery "Host wieder UP nach ..."). Intern ist der Service-Name `Host`, DOWN entspricht Status 2 und UNREACH Status 3 (für den Mindest-Schweregrad in Regeln); eine Regel mit Service-Muster `Host` trifft nur Host-Alarme. Die Host-Verzögerung (Alarmierung, Standard 0) ist unabhängig von Regel- und Standard-Verzögerung; Regel-Verzögerungen gelten nicht für Hosts. Solange ein Host nicht UP ist (auch in der Wartezeit), werden Service-Alarme dieses Hosts mit Grund "Host down" unterdrückt und nicht als gemeldet gezählt; bestehen sie nach der Rückkehr des Hosts weiter, greift die normale Verzögerung. Stummschaltung, Downtime und Bestätigung funktionieren für Hosts wie für Services. Das Dashboard zeigt Host-Probleme zuerst. +- Checkmk nicht erreichbar: Nach 3 aufeinanderfolgenden fehlgeschlagenen Abrufen kommt genau eine Systemmeldung an die Standardziele ("Checkmk nicht erreichbar seit HH:MM" bei Transportfehler, Timeout oder HTTP 5xx, "Antwort von Checkmk unerwartet seit HH:MM: ..." bei HTTP 4xx, ungültiger oder unerwarteter Antwort), beim ersten Erfolg danach "Checkmk wieder erreichbar nach ...". Die Meldung enthält nur einen kurzen Grund ohne Antwortinhalt und Secrets, läuft ohne KI und umgeht Verzögerung, Regeln und Stummschaltungen. Währenddessen gibt es keine Recoveries. Im Verlauf steht sie als Art `system` (Host `checkbot`, Service `Checkmk-Verbindung`). +- Sendelimit: höchstens N Nachrichten pro Minute und Ziel (Alarmierung, Standard 15, 1 bis 60, gleitendes Fenster je Backend und Ziel). Der Überschuss wird nicht einzeln gesendet, sondern als höchstens eine Sammelmeldung pro Minute und Ziel ("N weitere Alarme in der letzten Minute, siehe Verlauf", mit Link, wenn die Basis-URL gesetzt ist). Die betroffenen Einträge stehen im Verlauf als "unterdrückt (im Sammelhinweis)", die KI wird dafür nicht aufgerufen. KI-Nachträge zählen nicht gegen das Limit, Erinnerungen und Systemmeldungen schon; Systemmeldungen werden aber nie blockiert. +- Datenschutz je Backend: Telegram und Matrix haben die Einstellung "Inhalt der Alarmnachrichten": `voll` (Standard) oder `kurz` (Host, Service, Status und Verlaufszeile, ohne Plugin-Output und ohne KI-Text). Für `kurz`-Ziele wird keine KI-Analyse nachgereicht. Eine Überschreibung pro einzelnem Ziel gibt es nicht, weil Ziele in Regeln als freie Zeilen stehen; der Modus gilt je Backend. +- Link zur Oberfläche: Mit gesetzter "Basis-URL dieser Oberfläche" (Alarmierung, ohne abschließenden Schrägstrich gespeichert) enthält jede Alarmnachricht einen Link "Details" auf `/alerts#alert-` (Telegram als URL-Button in der Button-Reihe, Matrix als Zeile im Text). Lehnt Telegram die URL ab (z. B. localhost), wird die Nachricht ohne Link-Button gesendet. Leer = kein Link. + ### Web-UI - Anmeldung mit lokalen Nutzern (Argon2), optional TOTP. Rollen `admin` und `viewer`. -- Einstellungen mit Verbindungstest: Checkmk, Ollama, Telegram, Matrix, Checkmk Downtime. Dazu die Bereiche Alarmierung (Abfrageintervall, Standard-Verzögerung, Recovery-Schalter, Standard-Erinnerung, Bündelungsfenster, KI-Analysen pro Minute) und Aufbewahrung. +- Einstellungen mit Verbindungstest: Checkmk, Ollama, Telegram, Matrix, Checkmk Downtime. Dazu die Bereiche Alarmierung (Abfrageintervall, Standard-Verzögerung, Host-Verzögerung, Recovery-Schalter, Standard-Erinnerung, Bündelungsfenster, KI-Analysen pro Minute, Sendelimit, Basis-URL, Baseline-Button) und Aufbewahrung. - Setup-Assistent beim ersten Start, Dashboard mit Live-Status, Alarmverlauf, Audit-Log, Benutzerverwaltung. - Regeln und Stummschaltungen (Admins bearbeiten, Viewer sehen). Das Dashboard zeigt bei offenen Problemen den Meldestatus: "gemeldet", "wartet noch N min" oder "unterdrückt (Grund)". - Prompts (UI `/prompts`), siehe unten. Aufbewahrung (UI `/settings/retention`), siehe unten. @@ -109,10 +118,11 @@ Web-UI (FastAPI, Jinja2) ---- SQLite (Einstellungen, Verlauf, Audit, Nutzer) | Phase 5: Befehle, `mute`/`downtime`, Agent-Loop, Listener, Agent-Ansicht | Code geschrieben, ungetestet | | Phase 6: Prompt-Versionierung mit Rollback und Testbereich, Retention | Code geschrieben, ungetestet | | Phase 6b: Regeln (Ausgabefilter, Verzögerung, KI-Hinweis, Erinnerung, laut/leise), Delta, Erinnerungen, Recovery-Schalter, Zpool-Standardregel | Code geschrieben, ungetestet | +| Phase 6c: Baseline, Bestätigungen, Host-Alarme, Ausfallmeldung, Sendelimit, Datenschutz je Backend, UI-Link; atomare Migrationen; Erinnerung je Regel | Code geschrieben, ungetestet | | Phase 7: Deployment (`setup-lxc.sh`, Caddy, systemd, Backup, Checkmk-Local-Check) | Dateien geschrieben, nicht auf einem System ausprobiert | | Phase 8: Integrationstests | geplant | -Die Checkmk-Spaltennamen und der Downtime-Endpunkt sind Annahmen aus der Dokumentation und noch nicht an einer echten Instanz geprüft. +Die Checkmk-Spaltennamen und der Downtime-Endpunkt sind Annahmen aus der Dokumentation und noch nicht an einer echten Instanz geprüft. Das gilt ebenso für den Host-Abruf (`POST /domain-types/host/collections/all` mit den Spalten `name`, `state`, `plugin_output`, `last_state_change`, `scheduled_downtime_depth`, `acknowledged`) und die Service-Spalte `acknowledged`. ## Deployment @@ -164,6 +174,7 @@ Geht `master.key` verloren, sind alle in der Datenbank gespeicherten Secrets unb - Das Audit-Log enthält nie Secret-Werte. ## Entwicklung -- Tests: `pip install -r requirements-dev.txt`, dann `pytest`. +- Tests: `pip install -r requirements-dev.txt`, dann `pytest`. Für den ersten Lauf im Test-LXC gibt es `deploy/first-run.sh` (Dev-venv, Import- und Template-Prüfung, vollständiger pytest-Lauf, Archiv ohne Secrets), siehe `deploy/README.md`. +- Migrationen laufen je Migration in einer Transaktion mit Rollback; schlägt eine fehl, bleibt die Datenbank beim letzten vollständigen Stand und der Start bricht mit einer Fehlermeldung ab. - Der Plan und alle Entwurfsentscheidungen stehen in `CLAUDE.md`. - Code-Kommentare nur bei echtem WHY, Konstanten statt Magic Strings. diff --git a/app/alerts/__init__.py b/app/alerts/__init__.py index bf94cfc..f640ccb 100644 --- a/app/alerts/__init__.py +++ b/app/alerts/__init__.py @@ -1,5 +1,6 @@ from .analyzer import AlertAnalyzer, AnalysisResult, alert_model from .bundler import HostBundler +from .content import alert_link, base_url_provider, content_mode_provider, send_limit_provider from .delivery import Deliverer from .extras import AlertActions, Extras from .formatter import Analysis, format_message @@ -10,10 +11,13 @@ from .prompts import (DATA_NOTICE as ALERT_DATA_NOTICE, DEFAULT_ALERT_SYSTEM_PRO from .ratelimit import SlidingWindowLimiter from .report_status import ReportStatus, report_status from .repository import AlertRepository +from .sendlimit import SendLimiter +from .system import SystemNotifier __all__ = [ "ALERT_DATA_NOTICE", "DEFAULT_ALERT_SYSTEM_PROMPT", "alert_model", "build_messages", "AlertActions", "AlertAnalyzer", "AlertPipeline", "AlertRepository", "Analysis", "AnalysisResult", "Deliverer", "Extras", - "HostBundler", "ReportGate", "ReportStatus", "SlidingWindowLimiter", "build_system_prompt", - "build_user_prompt", "format_message", "report_status", "sanitize", + "HostBundler", "ReportGate", "ReportStatus", "SendLimiter", "SlidingWindowLimiter", "SystemNotifier", + "alert_link", "base_url_provider", "build_system_prompt", "build_user_prompt", "content_mode_provider", + "format_message", "report_status", "sanitize", "send_limit_provider", ] diff --git a/app/alerts/content.py b/app/alerts/content.py new file mode 100644 index 0000000..6c756fe --- /dev/null +++ b/app/alerts/content.py @@ -0,0 +1,31 @@ +from typing import Callable + +from ..chat import BACKEND_MATRIX, BACKEND_TELEGRAM +from ..constants import (ALERTS_SEND_LIMIT, CONTENT_MODE_FULL, CONTENT_MODE_SHORT, DEFAULT_SEND_LIMIT, + MATRIX_CONTENT_MODE, MAX_SEND_LIMIT, MIN_SEND_LIMIT, TELEGRAM_CONTENT_MODE, UI_BASE_URL) + +CONTENT_MODE_KEYS = {BACKEND_TELEGRAM: TELEGRAM_CONTENT_MODE, BACKEND_MATRIX: MATRIX_CONTENT_MODE} +ALERTS_PATH = "/alerts" +ALERT_ANCHOR = "alert-" + + +def content_mode_provider(store) -> Callable[[str], str]: + def mode(backend: str) -> str: + key = CONTENT_MODE_KEYS.get(backend) + return CONTENT_MODE_SHORT if key and store.get(key) == CONTENT_MODE_SHORT else CONTENT_MODE_FULL + return mode + + +def send_limit_provider(store) -> Callable[[], int]: + return lambda: min(store.get_int(ALERTS_SEND_LIMIT, DEFAULT_SEND_LIMIT, MIN_SEND_LIMIT), MAX_SEND_LIMIT) + + +def base_url_provider(store) -> Callable[[], str | None]: + return lambda: store.get(UI_BASE_URL) or None + + +def alert_link(base_url: str | None, alert_id: int | None = None) -> str | None: + if not base_url: + return None + base = base_url.rstrip("/") + return f"{base}{ALERTS_PATH}#{ALERT_ANCHOR}{alert_id}" if alert_id is not None else f"{base}{ALERTS_PATH}" diff --git a/app/alerts/delta.py b/app/alerts/delta.py index 4d695a9..91e6f80 100644 --- a/app/alerts/delta.py +++ b/app/alerts/delta.py @@ -1,6 +1,6 @@ from datetime import datetime -from ..models import KIND_PROBLEM, Event, state_name +from ..models import KIND_PROBLEM, Event, display_state, is_host_service MINUTE = 60 HOUR = 3600 @@ -9,6 +9,7 @@ CLOCK_FORMAT = "%H:%M" DATE_CLOCK_FORMAT = "%d.%m. %H:%M" ARROW = "→" LESS_THAN_MINUTE = "unter 1 min" +HOST_RECOVERY_HEAD = "Host wieder UP" def format_duration(seconds: int) -> str: @@ -31,10 +32,16 @@ def format_clock(ts: int, now: int) -> str: return moment.strftime(CLOCK_FORMAT if same_day else DATE_CLOCK_FORMAT) +def _name(event: Event, state: int) -> str: + return display_state(event.service, state) + + def state_label(event: Event) -> str: if event.previous_state is not None and event.previous_state != event.state and event.kind == KIND_PROBLEM: - return f"{state_name(event.previous_state)} {ARROW} {state_name(event.state)}" - return state_name(event.state) + label = f"{_name(event, event.previous_state)} {ARROW} {_name(event, event.state)}" + else: + label = _name(event, event.state) + return f"Host {label}" if is_host_service(event.service) else label def since_text(event: Event) -> str: @@ -42,21 +49,22 @@ def since_text(event: Event) -> str: return "" text = f"seit {format_clock(event.since_ts, event.ts)}" if event.previous_state is not None and event.previous_duration is not None: - text += f" (vorher {state_name(event.previous_state)} seit {format_duration(event.previous_duration)})" + text += f" (vorher {_name(event, event.previous_state)} seit {format_duration(event.previous_duration)})" return text def recovery_text(event: Event) -> str: - text = "wieder OK" + host = is_host_service(event.service) + text = HOST_RECOVERY_HEAD if host else "wieder OK" if event.problem_since_ts is not None: text += f" nach {format_duration(event.ts - event.problem_since_ts)}" if event.previous_state is not None: - text += f" (zuletzt {state_name(event.previous_state)})" + text += f" (zuletzt {_name(event, event.previous_state)})" return text def reminder_text(event: Event) -> str: - text = f"weiter {state_name(event.state)}" + text = f"weiter {_name(event, event.state)}" if event.since_ts is not None: text += f" seit {format_duration(event.ts - event.since_ts)}" return text diff --git a/app/alerts/extras.py b/app/alerts/extras.py index 33e0be6..a462a0a 100644 --- a/app/alerts/extras.py +++ b/app/alerts/extras.py @@ -2,14 +2,17 @@ from dataclasses import dataclass from typing import Callable, Sequence from ..chat import BACKEND_MATRIX, BACKEND_TELEGRAM, Button, Buttons, callback_data -from ..constants import CHECKMK_DT_ENABLED, FLAG_ON, MATRIX_ALLOWLIST, TELEGRAM_ALLOWLIST -from ..models import KIND_RECOVERY, Event +from ..constants import CHECKMK_DT_ENABLED, FLAG_ON, MATRIX_ALLOWLIST, TELEGRAM_ALLOWLIST, UI_BASE_URL +from ..models import KIND_RECOVERY, KIND_SYSTEM, Event, is_host_service +from .content import alert_link ALERT_BUTTON_MINUTES = 120 LABEL_MUTE = "2 h stumm" LABEL_DEEP = "Tiefenanalyse" LABEL_DOWNTIME = "Downtime 2 h" +LABEL_DETAILS = "Details" MATRIX_HINT = "Antwort mit !mute 2h zum Stummschalten" +MATRIX_LINK_PREFIX = "Details: " @dataclass(frozen=True) @@ -22,7 +25,8 @@ ExtrasProvider = Callable[[str], Extras | None] def actionable(events: Sequence[Event]) -> bool: - return len(events) == 1 and events[0].kind != KIND_RECOVERY and not events[0].suppressed + return (len(events) == 1 and events[0].kind not in (KIND_RECOVERY, KIND_SYSTEM) + and not events[0].suppressed) def telegram_buttons(alert_id: int, downtime: bool) -> Buttons: @@ -34,15 +38,34 @@ def telegram_buttons(alert_id: int, downtime: bool) -> Buttons: return tuple(rows) +def with_link(extras: Extras, backend: str, link: str | None) -> Extras: + if not link: + return extras + if backend == BACKEND_TELEGRAM: + button = Button(LABEL_DETAILS, url=link) + rows = (*extras.buttons[:-1], (*extras.buttons[-1], button)) if extras.buttons else ((button,),) + return Extras(buttons=rows, hint=extras.hint) + if backend == BACKEND_MATRIX: + line = f"{MATRIX_LINK_PREFIX}{link}" + return Extras(buttons=extras.buttons, hint=f"{extras.hint}\n{line}" if extras.hint else line) + return extras + + class AlertActions: def __init__(self, store): self._store = store def build(self, backend: str, events: Sequence[Event], alert_ids: Sequence[int]) -> Extras: + link = alert_link(self._store.get(UI_BASE_URL), alert_ids[0] if alert_ids else None) + return with_link(self._actions(backend, events, alert_ids), backend, link) + + def _actions(self, backend: str, events: Sequence[Event], alert_ids: Sequence[int]) -> Extras: if not actionable(events) or not alert_ids: return Extras() if backend == BACKEND_TELEGRAM and self._store.get(TELEGRAM_ALLOWLIST): - return Extras(buttons=telegram_buttons(alert_ids[0], self._store.get(CHECKMK_DT_ENABLED) == FLAG_ON)) + downtime = (self._store.get(CHECKMK_DT_ENABLED) == FLAG_ON + and not is_host_service(events[0].service)) + return Extras(buttons=telegram_buttons(alert_ids[0], downtime)) if backend == BACKEND_MATRIX and self._store.get(MATRIX_ALLOWLIST): return Extras(hint=MATRIX_HINT) return Extras() diff --git a/app/alerts/followup.py b/app/alerts/followup.py new file mode 100644 index 0000000..f27aa2b --- /dev/null +++ b/app/alerts/followup.py @@ -0,0 +1,46 @@ +import asyncio +from dataclasses import dataclass +from typing import Callable + +from ..chat import MessageRef +from ..models import AI_DONE, AI_FAILED, Event +from .analyzer import AlertAnalyzer +from .delivery import Deliverer +from .extras import ExtrasProvider +from .formatter import Analysis, format_message +from .repository import AlertRepository + +ExtrasFactory = Callable[[list[Event], list[int]], ExtrasProvider | None] + + +@dataclass +class Dispatch: + events: list[Event] + alert_ids: list[int] + refs: list[MessageRef] + silent: bool = False + + +class AnalysisFollowup: + def __init__(self, repo: AlertRepository, analyzer: AlertAnalyzer, deliverer: Deliverer, + extras: ExtrasFactory): + self._repo = repo + self._analyzer = analyzer + self._deliverer = deliverer + self._extras = extras + + async def run(self, dispatch: Dispatch) -> None: + analyses: dict[int, Analysis] = {} + for index, event in enumerate(dispatch.events): + if not event.wants_ai: + continue + result = await self._analyzer.analyze(event) + await asyncio.to_thread(self._repo.set_ai, dispatch.alert_ids[index], result.status, result.text) + if result.status == AI_DONE: + analyses[index] = Analysis(text=result.text) + elif result.status == AI_FAILED: + analyses[index] = Analysis(failed=True) + else: + continue + await self._deliverer.edit(dispatch.refs, format_message(dispatch.events, analyses), + self._extras(dispatch.events, dispatch.alert_ids), dispatch.silent) diff --git a/app/alerts/formatter.py b/app/alerts/formatter.py index 3775e80..cd077a3 100644 --- a/app/alerts/formatter.py +++ b/app/alerts/formatter.py @@ -3,8 +3,9 @@ from dataclasses import dataclass from html import escape from typing import Mapping, Sequence -from ..models import KIND_FLAPPING, KIND_RECOVERY, KIND_REMINDER, Event, state_name -from .delta import reminder_text, recovery_text, since_text, state_label +from ..constants import CONTENT_MODE_FULL +from ..models import KIND_FLAPPING, KIND_RECOVERY, KIND_REMINDER, Event, display_state, is_host_service +from .delta import HOST_RECOVERY_HEAD, reminder_text, recovery_text, since_text, state_label TELEGRAM_LIMIT = 4096 OUTPUT_MAX = 300 @@ -33,20 +34,33 @@ def clip_ai_text(text: str) -> str: def _label(event: Event) -> str: if event.kind == KIND_FLAPPING: - return f"FLAPPING ({state_name(event.state)})" + return f"FLAPPING ({display_state(event.service, event.state)})" return state_label(event) -def _event_line(event: Event) -> str: - since = since_text(event) if event.kind not in (KIND_RECOVERY, KIND_REMINDER, KIND_FLAPPING) else "" +def _detail(event: Event, full: bool) -> str: + output = clip(event.output, OUTPUT_MAX) if full else "" if event.kind == KIND_RECOVERY: - detail = recovery_text(event) - elif event.kind == KIND_REMINDER: - detail = f"{reminder_text(event)}: {clip(event.output, OUTPUT_MAX)}" - else: - detail = clip(event.output, OUTPUT_MAX) - head = f"{escape(_label(event))} {escape(event.service)}" - return f"{head}, {escape(since)}: {escape(detail)}" if since else f"{head}: {escape(detail)}" + return recovery_text(event) + if event.kind == KIND_REMINDER: + return f"{reminder_text(event)}: {output}" if full else reminder_text(event) + return output + + +def _host_recovery_line(event: Event) -> str: + return f"{escape(HOST_RECOVERY_HEAD)}{escape(recovery_text(event).removeprefix(HOST_RECOVERY_HEAD))}" + + +def _event_line(event: Event, full: bool = True) -> str: + host = is_host_service(event.service) + if host and event.kind == KIND_RECOVERY: + return _host_recovery_line(event) + since = since_text(event) if event.kind not in (KIND_RECOVERY, KIND_REMINDER, KIND_FLAPPING) else "" + detail = _detail(event, full) + label = f"{escape(_label(event))}" + head = label if host else f"{label} {escape(event.service)}" + head = f"{head}, {escape(since)}" if since else head + return f"{head}: {escape(detail)}" if (detail or full) else head def _analysis_block(event: Event, analysis: Analysis, single: bool) -> str: @@ -55,11 +69,13 @@ def _analysis_block(event: Event, analysis: Analysis, single: bool) -> str: return f"\n{title}\n{body}" -def format_message(events: Sequence[Event], analyses: Mapping[int, Analysis] | None = None) -> str: - analyses = analyses or {} +def format_message(events: Sequence[Event], analyses: Mapping[int, Analysis] | None = None, + mode: str = CONTENT_MODE_FULL) -> str: + full = mode == CONTENT_MODE_FULL + analyses = analyses if (analyses and full) else {} single = len(events) == 1 blocks = [f"{escape(events[0].host)}"] - blocks += [_event_line(event) for event in events] + blocks += [_event_line(event, full) for event in events] blocks += [_analysis_block(events[i], analyses[i], single) for i in sorted(analyses) if i < len(events)] return _fit(blocks) diff --git a/app/alerts/gate.py b/app/alerts/gate.py index b76c140..85fcdf1 100644 --- a/app/alerts/gate.py +++ b/app/alerts/gate.py @@ -1,6 +1,6 @@ import asyncio from dataclasses import replace -from typing import Callable +from typing import Callable, Iterable from ..models import (KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, KIND_REMINDER, SUPPRESS_NOT_REPORTED, SUPPRESS_RECOVERY_OFF, Event, Key) @@ -17,6 +17,9 @@ class ReportGate: self._counts = counts self._reported: dict[Key, bool] = {} + def reset(self, reported: Iterable[Key]) -> None: + self._reported = {key: True for key in reported} + async def apply(self, event: Event, recovery_enabled: bool) -> Event: if event.kind != KIND_RECOVERY or event.suppressed: return event diff --git a/app/alerts/pipeline.py b/app/alerts/pipeline.py index 014ad65..4cf5d74 100644 --- a/app/alerts/pipeline.py +++ b/app/alerts/pipeline.py @@ -2,20 +2,25 @@ import asyncio import logging import time import uuid -from dataclasses import dataclass, replace -from typing import Callable, Protocol +from dataclasses import replace +from typing import Callable, Iterable, Protocol -from ..chat import MessageRef, Target -from ..models import AI_DONE, AI_FAILED, AI_SKIPPED, Event +from ..chat import Target +from ..constants import CONTENT_MODE_FULL, CONTENT_MODE_SHORT, DEFAULT_SEND_LIMIT +from ..models import AI_SKIPPED, SUPPRESS_SEND_LIMIT, Event, Key from ..mutes import Mute, apply_mutes from ..routing import Route, ai_allowed, ai_hints, resolve_loud, resolve_targets from .analyzer import AlertAnalyzer from .bundler import HostBundler from .delivery import AdapterProvider, Deliverer from .extras import AlertActions -from .formatter import Analysis, format_message +from .followup import AnalysisFollowup, Dispatch +from .formatter import format_message from .gate import ReportGate from .repository import AlertRepository +from .sendlimit import SendLimiter +from .content import alert_link +from .system_text import overflow_message log = logging.getLogger(__name__) @@ -24,7 +29,7 @@ PRUNE_INTERVAL_SECONDS = 60.0 QUEUE_MAX = 1000 DRAIN_TIMEOUT_SECONDS = 15.0 -GroupKey = tuple[str, tuple[Target, ...], bool] +GroupKey = tuple[str, tuple[Target, ...], bool, tuple[str, ...]] class RouteSource(Protocol): @@ -37,20 +42,15 @@ class MuteSource(Protocol): def purge_expired(self, now: float | None = None) -> int: ... -@dataclass -class Dispatch: - events: list[Event] - alert_ids: list[int] - refs: list[MessageRef] - silent: bool = False - - class AlertPipeline: def __init__(self, repo: AlertRepository, adapters: AdapterProvider, analyzer: AlertAnalyzer, bundle_seconds: Callable[[], int], clock: Callable[[], float] = time.time, *, routes: RouteSource | None = None, mutes: MuteSource | None = None, actions: AlertActions | None = None, queue_size: int = QUEUE_MAX, - recovery_enabled: Callable[[], bool] = lambda: True): + recovery_enabled: Callable[[], bool] = lambda: True, limiter: SendLimiter | None = None, + send_limit: Callable[[], int] = lambda: DEFAULT_SEND_LIMIT, + content_mode: Callable[[str], str] = lambda backend: CONTENT_MODE_FULL, + base_url: Callable[[], str | None] = lambda: None): self._repo = repo self._adapters = adapters self._deliverer = Deliverer(adapters) @@ -61,6 +61,11 @@ class AlertPipeline: self._mutes = mutes self._actions = actions self._recovery_enabled = recovery_enabled + self._limiter = limiter or SendLimiter(clock) + self._send_limit = send_limit + self._content_mode = content_mode + self._base_url = base_url + self._followup = AnalysisFollowup(repo, analyzer, self._deliverer, self._extras) self._gate = ReportGate(repo.report_counts) self._bundler = HostBundler() self._queue: asyncio.Queue[Event] = asyncio.Queue(maxsize=queue_size) @@ -99,13 +104,16 @@ class AlertPipeline: except Exception: log.exception("Gebündelte Alarme konnten beim Beenden nicht gesendet werden") + def reset_reported(self, keys: Iterable[Key]) -> None: + self._gate.reset(keys) + async def flush_due(self) -> None: - for (_host, targets, silent), batch in self._bundler.due(self._clock(), self._bundle_seconds()): - await self._dispatch(batch, targets, silent) + for (_host, targets, silent, modes), batch in self._bundler.due(self._clock(), self._bundle_seconds()): + await self._dispatch(batch, targets, silent, modes) async def flush_all(self) -> None: - for (_host, targets, silent), batch in self._bundler.take_all(): - await self._dispatch(batch, targets, silent) + for (_host, targets, silent, modes), batch in self._bundler.take_all(): + await self._dispatch(batch, targets, silent, modes) async def wait_idle(self) -> None: while self._tasks: @@ -145,6 +153,7 @@ class AlertPipeline: await asyncio.sleep(FLUSH_TICK_SECONDS) try: await self.flush_due() + await self._flush_overflow() await self._prune_mutes() except asyncio.CancelledError: raise @@ -172,7 +181,8 @@ class AlertPipeline: targets = () if event.suppressed else tuple(sorted(resolve_targets(routes, event, defaults))) await self._gate.track(event, bool(targets)) if targets: - key = (event.host, targets, not resolve_loud(routes, event)) + modes = tuple(self._content_mode(target.backend) for target in targets) + key = (event.host, targets, not resolve_loud(routes, event), modes) groups.setdefault(key, []).append(event) else: stored_only.append(event) @@ -182,7 +192,7 @@ class AlertPipeline: for key, group in groups.items(): batch = self._bundler.submit(key, group, now, window) if batch: - await self._dispatch(batch, key[1], key[2]) + await self._dispatch(batch, key[1], key[2], key[3]) async def _store_only(self, events: list[Event]) -> None: ids = await asyncio.to_thread(self._repo.insert_alerts, events, uuid.uuid4().hex) @@ -193,36 +203,50 @@ class AlertPipeline: if event.wants_ai: await asyncio.to_thread(self._repo.set_ai, alert_id, AI_SKIPPED, None) - async def _dispatch(self, batch: list[Event], targets: tuple[Target, ...], silent: bool = False) -> None: + def _admit(self, targets: tuple[Target, ...], count: int) -> list[Target]: + limit = self._send_limit() + admitted = [] + for target in targets: + if self._limiter.acquire(target, limit): + admitted.append(target) + else: + self._limiter.overflowed(target, count) + return admitted + + async def _dispatch(self, batch: list[Event], targets: tuple[Target, ...], silent: bool = False, + modes: tuple[str, ...] = ()) -> None: + mode_of = dict(zip(targets, modes)) + admitted = self._admit(targets, len(batch)) + if not admitted: + await self._store_only([replace(e, suppressed=True, suppress_reason=SUPPRESS_SEND_LIMIT, + ai_enabled=False) for e in batch]) + return + short = [t for t in admitted if mode_of.get(t, CONTENT_MODE_FULL) == CONTENT_MODE_SHORT] + full = [t for t in admitted if t not in short] + if not full: + batch = [replace(e, ai_enabled=False) for e in batch] ids = await asyncio.to_thread(self._repo.insert_alerts, batch, uuid.uuid4().hex) - refs = await self._deliverer.send(format_message(batch), targets, self._extras(batch, ids), silent) - for ref in refs: + extras = self._extras(batch, ids) + refs = await self._deliverer.send(format_message(batch), full, extras, silent) if full else [] + short_refs = await self._deliverer.send( + format_message(batch, mode=CONTENT_MODE_SHORT), short, extras, silent) if short else [] + for ref in (*refs, *short_refs): for alert_id in ids: await asyncio.to_thread(self._repo.add_delivery, alert_id, ref) if refs and any(e.wants_ai for e in batch): - task = asyncio.create_task(self._analyze(Dispatch(batch, ids, refs, silent))) + task = asyncio.create_task(self._followup.run(Dispatch(batch, ids, refs, silent))) self._tasks.add(task) task.add_done_callback(self._tasks.discard) else: await self._skip_ai(batch, ids) + async def _flush_overflow(self) -> None: + for target, count in self._limiter.due_notices(): + self._limiter.acquire(target, self._send_limit(), force=True) + message = overflow_message(count, alert_link(self._base_url())) + await self._deliverer.send(message, [target]) + def _extras(self, events: list[Event], alert_ids: list[int]): if self._actions is None: return None return lambda backend: self._actions.build(backend, events, alert_ids) - - async def _analyze(self, dispatch: Dispatch) -> None: - analyses: dict[int, Analysis] = {} - for index, event in enumerate(dispatch.events): - if not event.wants_ai: - continue - result = await self._analyzer.analyze(event) - await asyncio.to_thread(self._repo.set_ai, dispatch.alert_ids[index], result.status, result.text) - if result.status == AI_DONE: - analyses[index] = Analysis(text=result.text) - elif result.status == AI_FAILED: - analyses[index] = Analysis(failed=True) - else: - continue - await self._deliverer.edit(dispatch.refs, format_message(dispatch.events, analyses), - self._extras(dispatch.events, dispatch.alert_ids), dispatch.silent) diff --git a/app/alerts/prompts.py b/app/alerts/prompts.py index 91652da..11e1bda 100644 --- a/app/alerts/prompts.py +++ b/app/alerts/prompts.py @@ -2,7 +2,7 @@ import re from datetime import datetime from typing import Sequence -from ..models import Event, state_name +from ..models import Event, display_state, state_name from ..poller.state import StateChange DATA_OPEN = "" @@ -62,7 +62,7 @@ def build_user_prompt(event: Event, history: Sequence[StateChange], hint: str = f"{DATA_OPEN}\n" f"Host: {sanitize(event.host, LABEL_MAX, True)}\n" f"Service: {sanitize(event.service, LABEL_MAX, True)}\n" - f"Status: {state_name(event.state)}\n" + f"Status: {display_state(event.service, event.state)}\n" f"Plugin-Output: {sanitize(event.output)}\n" f"Letzte Statuswechsel: {_history_line(history)}\n" f"Downtime: {in_downtime}\n" diff --git a/app/alerts/report_status.py b/app/alerts/report_status.py index 46cb383..abfc075 100644 --- a/app/alerts/report_status.py +++ b/app/alerts/report_status.py @@ -1,8 +1,8 @@ import math from dataclasses import dataclass -from typing import Mapping, Sequence +from typing import Collection, Mapping, Sequence -from ..models import STATE_OK +from ..models import HOST_SERVICE_NAME, STATE_OK, ack_holds from ..mutes import Mute, find_active_mute from ..routing import Route, resolve_delay @@ -21,14 +21,20 @@ class ReportStatus: def report_status(row: Mapping, rules: Sequence[Route], mutes: Sequence[Mute], default_minutes: int, - now: float) -> ReportStatus: + now: float, host_minutes: int = 0, down_hosts: Collection[str] = ()) -> ReportStatus: + is_host = row["service"] == HOST_SERVICE_NAME if row["in_downtime"]: return ReportStatus(STATUS_SUPPRESSED, "unterdrückt (Downtime)") if find_active_mute(mutes, row["host"], row["service"], now): return ReportStatus(STATUS_SUPPRESSED, "unterdrückt (Stummschaltung)") + if not is_host and row["host"] in down_hosts: + return ReportStatus(STATUS_SUPPRESSED, "unterdrückt (Host down)") + if ack_holds(row["state"], row.get("ack_state")): + return ReportStatus(STATUS_SUPPRESSED, "unterdrückt (bestätigt)") if row["flap_reported"] or (row["reported_state"] != STATE_OK and row["reported_state"] == row["state"]): return ReportStatus(STATUS_REPORTED, "gemeldet") - delay = resolve_delay(rules, row["host"], row["service"], row["state"], row["output"], default_minutes) + delay = host_minutes if is_host else resolve_delay( + rules, row["host"], row["service"], row["state"], row["output"], default_minutes) remaining = delay * SECONDS_PER_MINUTE - (now - row["last_state_change"]) if remaining <= 0: return ReportStatus(STATUS_DUE, "wird beim nächsten Abruf gemeldet") diff --git a/app/alerts/repository.py b/app/alerts/repository.py index f6115c9..4a46492 100644 --- a/app/alerts/repository.py +++ b/app/alerts/repository.py @@ -1,7 +1,7 @@ from ..chat import MessageRef from ..db import Database -from ..models import (AI_PENDING, AI_SKIPPED, KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, KIND_REMINDER, STATE_OK, - Event) +from ..models import (AI_PENDING, AI_SKIPPED, HOST_SERVICE_NAME, KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, + KIND_REMINDER, STATE_OK, Event) RECENT_LIMIT = 200 DAY_SECONDS = 86400 @@ -69,7 +69,8 @@ class AlertRepository: def open_problems(self) -> list[dict]: rows = self._db.query( "SELECT host, service, state, output, last_state_change, in_downtime, flapping, reported_state, " - "flap_reported " - "FROM service_states WHERE state!=? ORDER BY state DESC, host, service", (STATE_OK,), + "flap_reported, ack_state, host_held " + "FROM service_states WHERE state!=? ORDER BY (service=?) DESC, state DESC, host, service", + (STATE_OK, HOST_SERVICE_NAME), ) return [dict(row) for row in rows] diff --git a/app/alerts/sendlimit.py b/app/alerts/sendlimit.py new file mode 100644 index 0000000..9f29d22 --- /dev/null +++ b/app/alerts/sendlimit.py @@ -0,0 +1,38 @@ +import time +from collections import deque +from typing import Callable, Hashable + +WINDOW_SECONDS = 60 + + +class SendLimiter: + def __init__(self, clock: Callable[[], float] = time.time, window: float = WINDOW_SECONDS): + self._clock = clock + self._window = window + self._stamps: dict[Hashable, deque[float]] = {} + self._overflow: dict[Hashable, int] = {} + self._last_notice: dict[Hashable, float] = {} + + def acquire(self, key: Hashable, limit: int, force: bool = False) -> bool: + now = self._clock() + stamps = self._stamps.setdefault(key, deque()) + while stamps and now - stamps[0] >= self._window: + stamps.popleft() + if not force and len(stamps) >= limit: + return False + stamps.append(now) + return True + + def overflowed(self, key: Hashable, count: int = 1) -> None: + self._overflow[key] = self._overflow.get(key, 0) + count + + def due_notices(self) -> list[tuple[Hashable, int]]: + now = self._clock() + due = [] + for key, count in list(self._overflow.items()): + if count <= 0 or now - self._last_notice.get(key, float("-inf")) < self._window: + continue + self._last_notice[key] = now + self._overflow[key] = 0 + due.append((key, count)) + return due diff --git a/app/alerts/system.py b/app/alerts/system.py new file mode 100644 index 0000000..079b0e5 --- /dev/null +++ b/app/alerts/system.py @@ -0,0 +1,53 @@ +import asyncio +import time +import uuid +from typing import Callable, Iterable, Sequence + +from ..chat import Target +from ..models import KIND_SYSTEM, STATE_CRIT, STATE_OK, Event, Key, StateRow +from ..poller.outage import OutageNotice, RestoredNotice +from .delivery import AdapterProvider, Deliverer +from .extras import AlertActions +from .repository import AlertRepository +from .sendlimit import SendLimiter +from .system_text import (SERVICE_BASELINE, SERVICE_CHECKMK, SYSTEM_HOST, baseline_message, baseline_plain, + outage_plain, restored_plain, wrap) + + +class SystemNotifier: + def __init__(self, repo: AlertRepository, adapters: AdapterProvider, limiter: SendLimiter, + send_limit: Callable[[], int], actions: AlertActions | None = None, + on_baseline: Callable[[Iterable[Key]], None] | None = None, + clock: Callable[[], float] = time.time): + self._repo = repo + self._adapters = adapters + self._deliverer = Deliverer(adapters) + self._limiter = limiter + self._send_limit = send_limit + self._actions = actions + self._on_baseline = on_baseline + self._clock = clock + + async def outage(self, notice: OutageNotice) -> None: + await self._post(SERVICE_CHECKMK, STATE_CRIT, outage_plain(notice)) + + async def restored(self, notice: RestoredNotice) -> None: + await self._post(SERVICE_CHECKMK, STATE_OK, restored_plain(notice)) + + async def baseline(self, rows: Sequence[StateRow]) -> None: + if self._on_baseline is not None: + self._on_baseline([row.key for row in rows]) + await self._post(SERVICE_BASELINE, STATE_OK, baseline_plain(rows), baseline_message(rows)) + + async def _post(self, service: str, state: int, plain: str, message: str | None = None) -> None: + event = Event(ts=int(self._clock()), host=SYSTEM_HOST, service=service, kind=KIND_SYSTEM, state=state, + output=plain, ai_enabled=False) + ids = await asyncio.to_thread(self._repo.insert_alerts, [event], uuid.uuid4().hex) + targets = [Target(a.name, a.default_target) for a in await self._adapters()] + for target in targets: + self._limiter.acquire(target, self._send_limit(), force=True) + extras = (lambda backend: self._actions.build(backend, [event], ids)) if self._actions else None + refs = await self._deliverer.send(message or wrap(plain), targets, extras) + for ref in refs: + for alert_id in ids: + await asyncio.to_thread(self._repo.add_delivery, alert_id, ref) diff --git a/app/alerts/system_text.py b/app/alerts/system_text.py new file mode 100644 index 0000000..fc8bf74 --- /dev/null +++ b/app/alerts/system_text.py @@ -0,0 +1,87 @@ +from collections import Counter +from datetime import datetime +from html import escape +from typing import Sequence + +from ..checkmk import FAILURE_UNREACHABLE +from ..models import HOST_SERVICE_NAME, STATE_OK, StateRow, display_state, severity +from ..poller.outage import OutageNotice, RestoredNotice +from .delta import CLOCK_FORMAT, format_duration + +SYSTEM_HOST = "checkbot" +SERVICE_CHECKMK = "Checkmk-Verbindung" +SERVICE_BASELINE = "Baseline" +SERVICE_SEND_LIMIT = "Sendelimit" +TOP_COUNT = 10 +BULLET = "•" + + +def _clock(ts: int) -> str: + return datetime.fromtimestamp(ts).strftime(CLOCK_FORMAT) + + +def outage_plain(notice: OutageNotice) -> str: + since = _clock(notice.since) + if notice.kind == FAILURE_UNREACHABLE: + return f"Checkmk nicht erreichbar seit {since}" + return f"Antwort von Checkmk unerwartet seit {since}: {notice.brief}" + + +def restored_plain(notice: RestoredNotice) -> str: + return f"Checkmk wieder erreichbar nach {format_duration(notice.now - notice.since)}" + + +def wrap(plain: str) -> str: + return f"{escape(SYSTEM_HOST)}\n{escape(plain)}" + + +def _rank(row: StateRow) -> tuple: + return (row.service != HOST_SERVICE_NAME, -severity(row.state), row.host, row.service) + + +def _count_line(title: str, items: Sequence[StateRow], service: str) -> str: + counts = Counter(r.state for r in items) + parts = [f"{display_state(service, state)} {number}" + for state, number in sorted(counts.items(), key=lambda item: -severity(item[0]))] + return f"{title}: {', '.join(parts) or 'keine'}" + + +def _counts(rows: Sequence[StateRow]) -> list[str]: + hosts = [r for r in rows if r.service == HOST_SERVICE_NAME] + services = [r for r in rows if r.service != HOST_SERVICE_NAME] + return [_count_line("Hosts", hosts, HOST_SERVICE_NAME), _count_line("Services", services, "")] + + +def _top_line(row: StateRow) -> str: + if row.service == HOST_SERVICE_NAME: + return f"{BULLET} {escape(row.host)} Host {escape(display_state(row.service, row.state))}" + state = escape(display_state(row.service, row.state)) + return f"{BULLET} {escape(row.host)} {escape(row.service)}: {state}" + + +def baseline_plain(rows: Sequence[StateRow]) -> str: + return f"Baseline aufgenommen: {len([r for r in rows if r.state != STATE_OK])} offene Probleme" + + +def baseline_message(rows: Sequence[StateRow]) -> str: + open_ = sorted((r for r in rows if r.state != STATE_OK), key=_rank) + blocks = [f"{escape(SYSTEM_HOST)}", + f"{escape(baseline_plain(open_))}", + "Still aufgenommen, bestehende Probleme werden nicht einzeln gemeldet; Recoveries kommen später.", + *(escape(line) for line in _counts(open_))] + if open_: + blocks.append(f"Die {min(TOP_COUNT, len(open_))} kritischsten:") + blocks += [_top_line(row) for row in open_[:TOP_COUNT]] + if len(open_) > TOP_COUNT: + blocks.append(escape(f"… und {len(open_) - TOP_COUNT} weitere")) + return "\n".join(blocks) + + +def overflow_plain(count: int) -> str: + return f"{count} weitere Alarme in der letzten Minute, siehe Verlauf" if count != 1 else \ + "1 weiterer Alarm in der letzten Minute, siehe Verlauf" + + +def overflow_message(count: int, link: str | None = None) -> str: + text = wrap(overflow_plain(count)) + return f"{text}\n{escape(link)}" if link else text diff --git a/app/chat/base.py b/app/chat/base.py index 0791706..8573172 100644 --- a/app/chat/base.py +++ b/app/chat/base.py @@ -30,7 +30,8 @@ def json_body(response: httpx.Response) -> dict: @dataclass(frozen=True) class Button: label: str - data: str + data: str = "" + url: str = "" Buttons = tuple[tuple[Button, ...], ...] diff --git a/app/chat/telegram.py b/app/chat/telegram.py index 48dd497..1d99ed6 100644 --- a/app/chat/telegram.py +++ b/app/chat/telegram.py @@ -3,7 +3,7 @@ import asyncio import httpx from ..constants import DEFAULT_TELEGRAM_API_BASE -from .base import Buttons, ChatError, MessageRef, json_body, mask_secret +from .base import Button, Buttons, ChatError, MessageRef, json_body, mask_secret BACKEND_TELEGRAM = "telegram" PARSE_MODE = "HTML" @@ -17,8 +17,20 @@ ANSWER_MAX = 200 ERROR_MAX = 300 +URL_BUTTON_ERRORS = ("button_url_invalid", "keyboard button url") + + +def _button(b: Button) -> dict: + return {"text": b.label, "url": b.url} if b.url else {"text": b.label, "callback_data": b.data} + + def _markup(buttons: Buttons) -> dict: - return {"inline_keyboard": [[{"text": b.label, "callback_data": b.data} for b in row] for row in buttons]} + return {"inline_keyboard": [[_button(b) for b in row] for row in buttons]} + + +def _without_url_buttons(buttons: Buttons) -> Buttons: + rows = (tuple(b for b in row if not b.url) for row in buttons) + return tuple(row for row in rows if row) class TelegramError(ChatError): @@ -58,6 +70,19 @@ class TelegramAdapter: raise TelegramError(self._mask(f"HTTP {response.status_code}: {description}")[:ERROR_MAX]) raise TelegramError("Telegram nicht erreichbar (Rate-Limit)") + async def _call_buttons(self, method: str, payload: dict, buttons: Buttons) -> dict: + try: + return await self._call(method, payload) + except TelegramError as exc: + # Telegram lehnt manche URLs (z. B. localhost) ab; ohne Link-Button soll der Alarm trotzdem ankommen. + if not any(m in str(exc).lower() for m in URL_BUTTON_ERRORS) or not any(b.url for row in buttons for b in row): + raise + remaining = _without_url_buttons(buttons) + payload = {k: v for k, v in payload.items() if k != "reply_markup"} + if remaining: + payload["reply_markup"] = _markup(remaining) + return await self._call(method, payload) + async def send(self, target: str, text: str, buttons: Buttons = (), reply_to: str | None = None, html: bool = True, silent: bool = False) -> MessageRef: payload: dict = {"chat_id": target, "text": text, "disable_web_page_preview": True} @@ -69,7 +94,7 @@ class TelegramAdapter: payload["reply_markup"] = _markup(buttons) if reply_to: payload["reply_parameters"] = {"message_id": int(reply_to), "allow_sending_without_reply": True} - result = await self._call("sendMessage", payload) + result = await self._call_buttons("sendMessage", payload, buttons) return MessageRef(self.name, str(target), str(result["message_id"])) async def edit(self, ref: MessageRef, text: str, buttons: Buttons = (), silent: bool = False) -> None: @@ -78,7 +103,7 @@ class TelegramAdapter: if buttons: payload["reply_markup"] = _markup(buttons) try: - await self._call("editMessageText", payload) + await self._call_buttons("editMessageText", payload, buttons) except TelegramError as exc: if NOT_MODIFIED not in str(exc).lower(): raise diff --git a/app/checkmk.py b/app/checkmk.py index 9a87905..33ab364 100644 --- a/app/checkmk.py +++ b/app/checkmk.py @@ -4,9 +4,11 @@ import httpx from .constants import (CHECKMK_CA_FILE, CHECKMK_SECRET, CHECKMK_SITE, CHECKMK_URL, CHECKMK_USER, CHECKMK_VERIFY_TLS, FLAG_OFF, FLAG_ON) +from .models import HOST_SERVICE_NAME API_PATH = "/check_mk/api/1.0" SERVICES_PATH = "/domain-types/service/collections/all" +HOSTS_PATH = "/domain-types/host/collections/all" VERSION_PATH = "/version" DOWNTIMES_PATH = "/domain-types/downtime/collections/all" DOWNTIME_TYPE_PARAM = "downtime_type" @@ -25,13 +27,38 @@ COLUMNS = [ "last_state_change", "scheduled_downtime_depth", "is_flapping", + "acknowledged", ] +HOST_COLUMNS = [ + "name", + "state", + "plugin_output", + "last_state_change", + "scheduled_downtime_depth", + "acknowledged", +] +# Checkmk: 0 UP, 1 DOWN, 2 UNREACH. Intern DOWN=2 und UNREACH=3, damit min_state-Regeln wie bei Services greifen. +HOST_STATE_MAP = {0: 0, 1: 2, 2: 3} +HOST_STATE_FALLBACK = 2 + +FAILURE_UNREACHABLE = "unreachable" +FAILURE_UNEXPECTED = "unexpected" + NOT_OK_QUERY = {"op": "!=", "left": "state", "right": str(STATE_OK)} class CheckmkError(Exception): - pass + kind = FAILURE_UNEXPECTED + + def __init__(self, message: str, brief: str = ""): + super().__init__(message) + # Kurztext ohne Antwortinhalt, damit er gefahrlos in Chat-Meldungen steht. + self.brief = brief or type(self).__name__ + + +class CheckmkUnreachable(CheckmkError): + kind = FAILURE_UNREACHABLE @dataclass(frozen=True) @@ -44,6 +71,21 @@ class ServiceState: last_state_change: int in_downtime: bool flapping: bool + acknowledged: bool = False + + +@dataclass(frozen=True) +class HostState: + host: str + state: int + output: str + last_state_change: int + in_downtime: bool + acknowledged: bool + + def as_service(self) -> ServiceState: + return ServiceState(self.host, HOST_SERVICE_NAME, self.state, True, self.output, + self.last_state_change, self.in_downtime, False, self.acknowledged) @dataclass(frozen=True) @@ -85,9 +127,29 @@ def _parse_service(extensions: dict) -> ServiceState: last_state_change=int(extensions.get("last_state_change", 0)), in_downtime=int(extensions.get("scheduled_downtime_depth", 0)) > 0, flapping=bool(int(extensions.get("is_flapping", 0))), + acknowledged=bool(int(extensions.get("acknowledged", 0))), ) +def _parse_host(extensions: dict) -> HostState: + return HostState( + host=extensions["name"], + state=HOST_STATE_MAP.get(int(extensions["state"]), HOST_STATE_FALLBACK), + output=extensions.get("plugin_output", ""), + last_state_change=int(extensions.get("last_state_change", 0)), + in_downtime=int(extensions.get("scheduled_downtime_depth", 0)) > 0, + acknowledged=bool(int(extensions.get("acknowledged", 0))), + ) + + +def _items(body, parser) -> list: + try: + return [parser(item["extensions"]) for item in body.get("value", [])] + except (KeyError, TypeError, ValueError, AttributeError) as exc: + raise CheckmkError(f"Unerwartetes Antwortformat ({type(exc).__name__})", + brief="Antwortformat unerwartet") from exc + + def is_configured(store) -> bool: return all(store.get(key) for key in (CHECKMK_URL, CHECKMK_SITE, CHECKMK_USER, CHECKMK_SECRET)) @@ -120,10 +182,15 @@ class CheckmkClient: try: response = await self._client.request(method, path, **kwargs) except httpx.HTTPError as exc: - raise CheckmkError(f"{type(exc).__name__}: {exc}") from exc - if response.status_code >= 400: - raise CheckmkError(f"HTTP {response.status_code} on {path}: {response.text[:200]}") - return response.json() + raise CheckmkUnreachable(f"{type(exc).__name__}: {exc}", brief=type(exc).__name__) from exc + status = response.status_code + if status >= 400: + error = CheckmkUnreachable if status >= 500 else CheckmkError + raise error(f"HTTP {status} on {path}: {response.text[:200]}", brief=f"HTTP {status}") + try: + return response.json() + except ValueError as exc: + raise CheckmkError(f"Antwort auf {path} ist kein JSON", brief="Antwort ist kein JSON") from exc async def version(self) -> dict: return await self._request("GET", VERSION_PATH) @@ -134,14 +201,22 @@ class CheckmkClient: SERVICES_PATH, json={"query": NOT_OK_QUERY, "columns": COLUMNS}, ) - return [_parse_service(item["extensions"]) for item in body.get("value", [])] + return _items(body, _parse_service) + + async def problem_hosts(self) -> list[HostState]: + body = await self._request( + "POST", + HOSTS_PATH, + json={"query": NOT_OK_QUERY, "columns": HOST_COLUMNS}, + ) + return _items(body, _parse_host) async def problems(self) -> list[ServiceState]: return await self.problem_services() async def _services(self, query: dict) -> list[ServiceState]: body = await self._request("POST", SERVICES_PATH, json={"query": query, "columns": COLUMNS}) - return [_parse_service(item["extensions"]) for item in body.get("value", [])] + return _items(body, _parse_service) async def service_status(self, host: str, service: str) -> ServiceState | None: query = {"op": "and", "expr": [_equals("host_name", host), _equals("description", service)]} @@ -155,7 +230,7 @@ class CheckmkClient: downtimes: list[Downtime] = [] for downtime_type in DOWNTIME_TYPES: body = await self._request("GET", DOWNTIMES_PATH, params={DOWNTIME_TYPE_PARAM: downtime_type}) - downtimes += [_parse_downtime(item["extensions"]) for item in body.get("value", [])] + downtimes += _items(body, _parse_downtime) if host: wanted = host.casefold() downtimes = [d for d in downtimes if d.host.casefold() == wanted] diff --git a/app/constants.py b/app/constants.py index 7a751d6..43eaab6 100644 --- a/app/constants.py +++ b/app/constants.py @@ -32,6 +32,12 @@ MATRIX_USER_ID = "matrix.user_id" MATRIX_TOKEN = "matrix.token" MATRIX_ROOM_ID = "matrix.room_id" MATRIX_ALLOWLIST = "matrix.allowlist" +TELEGRAM_CONTENT_MODE = "telegram.content_mode" +MATRIX_CONTENT_MODE = "matrix.content_mode" + +CONTENT_MODE_FULL = "voll" +CONTENT_MODE_SHORT = "kurz" +CONTENT_MODES = (CONTENT_MODE_FULL, CONTENT_MODE_SHORT) ADAPTER_SETTING_KEYS = ( TELEGRAM_ENABLED, TELEGRAM_TOKEN, TELEGRAM_CHAT_ID, TELEGRAM_API_BASE, @@ -44,6 +50,9 @@ ALERTS_AI_PER_MINUTE = "alerts.ai_per_minute" ALERTS_DEFAULT_DELAY_MINUTES = "alerts.default_delay_minutes" ALERTS_RECOVERY_ENABLED = "alerts.recovery_enabled" ALERTS_REMINDER_HOURS = "alerts.reminder_hours" +ALERTS_HOST_DELAY_MINUTES = "alerts.host_delay_minutes" +ALERTS_SEND_LIMIT = "alerts.send_limit_per_minute" +UI_BASE_URL = "ui.base_url" PROMPT_ALERT_SYSTEM = "prompt.alert_system" DEFAULT_POLL_INTERVAL = 30 @@ -54,6 +63,10 @@ DEFAULT_DELAY_MINUTES = 15 MAX_DELAY_MINUTES = 1440 DEFAULT_REMINDER_HOURS = 0 MAX_REMINDER_HOURS = 168 +DEFAULT_HOST_DELAY_MINUTES = 0 +DEFAULT_SEND_LIMIT = 15 +MIN_SEND_LIMIT = 1 +MAX_SEND_LIMIT = 60 DEFAULT_TELEGRAM_API_BASE = "https://api.telegram.org" CHECKMK_DT_ENABLED = "checkmk_dt.enabled" diff --git a/app/db.py b/app/db.py index 1d54bf2..5853764 100644 --- a/app/db.py +++ b/app/db.py @@ -1,176 +1,16 @@ import sqlite3 import threading from pathlib import Path +from typing import Sequence -MIGRATIONS = [ - """ - CREATE TABLE settings ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL, - is_secret INTEGER NOT NULL DEFAULT 0, - updated_at INTEGER NOT NULL - ); - CREATE TABLE audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ts INTEGER NOT NULL, - user TEXT NOT NULL, - action TEXT NOT NULL, - target TEXT NOT NULL, - old_value TEXT, - new_value TEXT - ); - CREATE INDEX audit_ts ON audit(ts); - """, - """ - CREATE TABLE users ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - username TEXT NOT NULL UNIQUE COLLATE NOCASE, - password_hash TEXT NOT NULL, - role TEXT NOT NULL CHECK (role IN ('admin', 'viewer')), - totp_secret TEXT, - session_version INTEGER NOT NULL DEFAULT 0, - disabled INTEGER NOT NULL DEFAULT 0, - created_at INTEGER NOT NULL - ); - """, - """ - CREATE TABLE service_states ( - host TEXT NOT NULL, - service TEXT NOT NULL, - state INTEGER NOT NULL, - output TEXT NOT NULL DEFAULT '', - last_state_change INTEGER NOT NULL, - in_downtime INTEGER NOT NULL DEFAULT 0, - flapping INTEGER NOT NULL DEFAULT 0, - reported_state INTEGER NOT NULL DEFAULT 0, - flap_reported INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (host, service) - ); - CREATE TABLE state_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - host TEXT NOT NULL, - service TEXT NOT NULL, - state INTEGER NOT NULL, - ts INTEGER NOT NULL - ); - CREATE INDEX state_log_lookup ON state_log(host, service, ts); - CREATE TABLE alerts ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ts INTEGER NOT NULL, - host TEXT NOT NULL, - service TEXT NOT NULL, - kind TEXT NOT NULL, - state INTEGER NOT NULL, - output TEXT NOT NULL DEFAULT '', - suppressed INTEGER NOT NULL DEFAULT 0, - suppress_reason TEXT, - ai_status TEXT NOT NULL DEFAULT 'skipped' - CHECK (ai_status IN ('pending', 'done', 'failed', 'skipped')), - ai_text TEXT, - message_group TEXT NOT NULL - ); - CREATE INDEX alerts_ts ON alerts(ts); - CREATE INDEX alerts_group ON alerts(message_group); - CREATE TABLE deliveries ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - alert_id INTEGER NOT NULL REFERENCES alerts(id) ON DELETE CASCADE, - backend TEXT NOT NULL, - target TEXT NOT NULL, - message_id TEXT NOT NULL - ); - CREATE INDEX deliveries_alert ON deliveries(alert_id); - """, - """ - CREATE TABLE routes ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - position INTEGER NOT NULL, - enabled INTEGER NOT NULL DEFAULT 1, - match_host TEXT NOT NULL DEFAULT '', - match_service TEXT NOT NULL DEFAULT '', - min_state INTEGER NOT NULL DEFAULT 1 CHECK (min_state BETWEEN 0 AND 3), - kinds TEXT NOT NULL DEFAULT 'problem,recovery,flapping', - stop_on_match INTEGER NOT NULL DEFAULT 0, - targets TEXT NOT NULL DEFAULT '[]' - ); - CREATE INDEX routes_position ON routes(position); - CREATE TABLE mutes ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - host_glob TEXT NOT NULL, - service_glob TEXT NOT NULL, - expires_at INTEGER NOT NULL, - reason TEXT NOT NULL DEFAULT '', - created_by TEXT NOT NULL, - created_at INTEGER NOT NULL, - source TEXT NOT NULL CHECK (source IN ('ui', 'chat')) - ); - CREATE INDEX mutes_expires ON mutes(expires_at); - """, - """ - CREATE TABLE agent_runs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ts INTEGER NOT NULL, - backend TEXT NOT NULL, - sender TEXT NOT NULL, - command TEXT NOT NULL, - question TEXT NOT NULL, - answer TEXT NOT NULL, - abort_reason TEXT, - steps TEXT NOT NULL DEFAULT '[]' - ); - CREATE INDEX agent_runs_ts ON agent_runs(ts); - CREATE TABLE listener_state ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - """, - """ - CREATE TABLE prompt_versions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - version INTEGER NOT NULL, - text TEXT NOT NULL, - note TEXT NOT NULL DEFAULT '', - created_by TEXT NOT NULL, - created_at INTEGER NOT NULL, - UNIQUE (name, version) - ); - CREATE TABLE prompt_active ( - name TEXT PRIMARY KEY, - version INTEGER NOT NULL - ); - CREATE TABLE retention_state ( - id INTEGER PRIMARY KEY CHECK (id = 1), - ts INTEGER NOT NULL, - origin TEXT NOT NULL, - counts TEXT NOT NULL - ); - CREATE INDEX state_log_ts ON state_log(ts); - """, - """ - ALTER TABLE service_states ADD COLUMN problem_since INTEGER; - ALTER TABLE service_states ADD COLUMN reported_at INTEGER NOT NULL DEFAULT 0; - ALTER TABLE service_states ADD COLUMN reported_since INTEGER NOT NULL DEFAULT 0; - ALTER TABLE service_states ADD COLUMN last_reminder_ts INTEGER NOT NULL DEFAULT 0; - UPDATE service_states SET reported_at = CAST(strftime('%s', 'now') AS INTEGER), - reported_since = last_state_change WHERE reported_state != 0; - ALTER TABLE routes ADD COLUMN delay_minutes INTEGER; - ALTER TABLE routes ADD COLUMN output_contains TEXT NOT NULL DEFAULT ''; - ALTER TABLE routes ADD COLUMN ai_enabled INTEGER NOT NULL DEFAULT 1; - ALTER TABLE routes ADD COLUMN ai_hint TEXT NOT NULL DEFAULT ''; - ALTER TABLE routes ADD COLUMN reminder_hours INTEGER; - ALTER TABLE routes ADD COLUMN loud INTEGER NOT NULL DEFAULT 1; - ALTER TABLE routes ADD COLUMN use_default_targets INTEGER NOT NULL DEFAULT 0; - CREATE TABLE app_state ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - """, -] +from .migrate import MigrationError, run_migrations +from .migrations import MIGRATIONS + +__all__ = ["Database", "MIGRATIONS", "MigrationError"] class Database: - def __init__(self, path): + def __init__(self, path, migrations: Sequence[str] | None = None): if str(path) != ":memory:": Path(path).parent.mkdir(parents=True, exist_ok=True) self._conn = sqlite3.connect(str(path), check_same_thread=False) @@ -178,15 +18,16 @@ class Database: self._conn.execute("PRAGMA journal_mode=WAL") self._conn.execute("PRAGMA foreign_keys=ON") self._lock = threading.Lock() - self._migrate() + self._migrations = MIGRATIONS if migrations is None else list(migrations) + try: + self._migrate() + except MigrationError: + self._conn.close() + raise def _migrate(self) -> None: with self._lock: - version = self._conn.execute("PRAGMA user_version").fetchone()[0] - for index in range(version, len(MIGRATIONS)): - self._conn.executescript(MIGRATIONS[index]) - self._conn.execute(f"PRAGMA user_version={index + 1}") - self._conn.commit() + run_migrations(self._conn, self._migrations) def execute(self, sql: str, params: tuple = ()) -> int: with self._lock: diff --git a/app/main.py b/app/main.py index a2763c3..ac0b5c8 100644 --- a/app/main.py +++ b/app/main.py @@ -17,8 +17,8 @@ from .ratelimit import FailureLimiter from .runtime import Runtime from .settings import SettingsStore from .users import UserStore -from .web import (account, agent_pages, alerts, auth, matrix_section, mutes_pages, pages, prompts_pages, - prompts_test_pages, retention_pages, routing_pages, telegram_section) +from .web import (account, agent_pages, alerts, auth, baseline_pages, matrix_section, mutes_pages, pages, + prompts_pages, prompts_test_pages, retention_pages, routing_pages, telegram_section) from .web.deps import LoginRequired, redirect, templates, verify_csrf STATIC_DIR = Path(__file__).parent / "web" / "static" @@ -83,6 +83,7 @@ def create_app(config: FileConfig | None = None) -> FastAPI: app.include_router(account.router) app.include_router(telegram_section.router) app.include_router(matrix_section.router) + app.include_router(baseline_pages.router) app.include_router(routing_pages.router) app.include_router(mutes_pages.router) app.include_router(alerts.router) diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 0000000..2a77856 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,46 @@ +import sqlite3 +from typing import Sequence + + +class MigrationError(Exception): + pass + + +def split_statements(script: str) -> list[str]: + statements: list[str] = [] + buffer = "" + for line in script.splitlines(keepends=True): + buffer += line + if sqlite3.complete_statement(buffer): + statements.append(buffer.strip()) + buffer = "" + if buffer.strip(): + statements.append(buffer.strip()) + return statements + + +def apply_migration(conn: sqlite3.Connection, index: int, script: str) -> None: + try: + conn.execute("BEGIN IMMEDIATE") + for statement in split_statements(script): + conn.execute(statement) + conn.execute(f"PRAGMA user_version={index + 1}") + conn.execute("COMMIT") + except Exception as exc: + try: + conn.execute("ROLLBACK") + except sqlite3.Error: + pass + raise MigrationError(f"Migration {index} fehlgeschlagen und zurückgerollt: {exc}") from exc + + +def run_migrations(conn: sqlite3.Connection, migrations: Sequence[str]) -> None: + version = conn.execute("PRAGMA user_version").fetchone()[0] + # Im Autocommit-Modus, damit BEGIN/COMMIT ausschließlich hier gesetzt werden (executescript würde implizit committen). + previous = conn.isolation_level + conn.isolation_level = None + try: + for index in range(version, len(migrations)): + apply_migration(conn, index, migrations[index]) + finally: + conn.isolation_level = previous diff --git a/app/migrations.py b/app/migrations.py new file mode 100644 index 0000000..e2aa9a5 --- /dev/null +++ b/app/migrations.py @@ -0,0 +1,174 @@ +MIGRATIONS = [ + """ + CREATE TABLE settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + is_secret INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL + ); + CREATE TABLE audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER NOT NULL, + user TEXT NOT NULL, + action TEXT NOT NULL, + target TEXT NOT NULL, + old_value TEXT, + new_value TEXT + ); + CREATE INDEX audit_ts ON audit(ts); + """, + """ + CREATE TABLE users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + username TEXT NOT NULL UNIQUE COLLATE NOCASE, + password_hash TEXT NOT NULL, + role TEXT NOT NULL CHECK (role IN ('admin', 'viewer')), + totp_secret TEXT, + session_version INTEGER NOT NULL DEFAULT 0, + disabled INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL + ); + """, + """ + CREATE TABLE service_states ( + host TEXT NOT NULL, + service TEXT NOT NULL, + state INTEGER NOT NULL, + output TEXT NOT NULL DEFAULT '', + last_state_change INTEGER NOT NULL, + in_downtime INTEGER NOT NULL DEFAULT 0, + flapping INTEGER NOT NULL DEFAULT 0, + reported_state INTEGER NOT NULL DEFAULT 0, + flap_reported INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (host, service) + ); + CREATE TABLE state_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + host TEXT NOT NULL, + service TEXT NOT NULL, + state INTEGER NOT NULL, + ts INTEGER NOT NULL + ); + CREATE INDEX state_log_lookup ON state_log(host, service, ts); + CREATE TABLE alerts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER NOT NULL, + host TEXT NOT NULL, + service TEXT NOT NULL, + kind TEXT NOT NULL, + state INTEGER NOT NULL, + output TEXT NOT NULL DEFAULT '', + suppressed INTEGER NOT NULL DEFAULT 0, + suppress_reason TEXT, + ai_status TEXT NOT NULL DEFAULT 'skipped' + CHECK (ai_status IN ('pending', 'done', 'failed', 'skipped')), + ai_text TEXT, + message_group TEXT NOT NULL + ); + CREATE INDEX alerts_ts ON alerts(ts); + CREATE INDEX alerts_group ON alerts(message_group); + CREATE TABLE deliveries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + alert_id INTEGER NOT NULL REFERENCES alerts(id) ON DELETE CASCADE, + backend TEXT NOT NULL, + target TEXT NOT NULL, + message_id TEXT NOT NULL + ); + CREATE INDEX deliveries_alert ON deliveries(alert_id); + """, + """ + CREATE TABLE routes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + position INTEGER NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1, + match_host TEXT NOT NULL DEFAULT '', + match_service TEXT NOT NULL DEFAULT '', + min_state INTEGER NOT NULL DEFAULT 1 CHECK (min_state BETWEEN 0 AND 3), + kinds TEXT NOT NULL DEFAULT 'problem,recovery,flapping', + stop_on_match INTEGER NOT NULL DEFAULT 0, + targets TEXT NOT NULL DEFAULT '[]' + ); + CREATE INDEX routes_position ON routes(position); + CREATE TABLE mutes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + host_glob TEXT NOT NULL, + service_glob TEXT NOT NULL, + expires_at INTEGER NOT NULL, + reason TEXT NOT NULL DEFAULT '', + created_by TEXT NOT NULL, + created_at INTEGER NOT NULL, + source TEXT NOT NULL CHECK (source IN ('ui', 'chat')) + ); + CREATE INDEX mutes_expires ON mutes(expires_at); + """, + """ + CREATE TABLE agent_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER NOT NULL, + backend TEXT NOT NULL, + sender TEXT NOT NULL, + command TEXT NOT NULL, + question TEXT NOT NULL, + answer TEXT NOT NULL, + abort_reason TEXT, + steps TEXT NOT NULL DEFAULT '[]' + ); + CREATE INDEX agent_runs_ts ON agent_runs(ts); + CREATE TABLE listener_state ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + """, + """ + CREATE TABLE prompt_versions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + version INTEGER NOT NULL, + text TEXT NOT NULL, + note TEXT NOT NULL DEFAULT '', + created_by TEXT NOT NULL, + created_at INTEGER NOT NULL, + UNIQUE (name, version) + ); + CREATE TABLE prompt_active ( + name TEXT PRIMARY KEY, + version INTEGER NOT NULL + ); + CREATE TABLE retention_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + ts INTEGER NOT NULL, + origin TEXT NOT NULL, + counts TEXT NOT NULL + ); + CREATE INDEX state_log_ts ON state_log(ts); + """, + """ + ALTER TABLE service_states ADD COLUMN problem_since INTEGER; + ALTER TABLE service_states ADD COLUMN reported_at INTEGER NOT NULL DEFAULT 0; + ALTER TABLE service_states ADD COLUMN reported_since INTEGER NOT NULL DEFAULT 0; + ALTER TABLE service_states ADD COLUMN last_reminder_ts INTEGER NOT NULL DEFAULT 0; + UPDATE service_states SET reported_at = CAST(strftime('%s', 'now') AS INTEGER), + reported_since = last_state_change WHERE reported_state != 0; + ALTER TABLE routes ADD COLUMN delay_minutes INTEGER; + ALTER TABLE routes ADD COLUMN output_contains TEXT NOT NULL DEFAULT ''; + ALTER TABLE routes ADD COLUMN ai_enabled INTEGER NOT NULL DEFAULT 1; + ALTER TABLE routes ADD COLUMN ai_hint TEXT NOT NULL DEFAULT ''; + ALTER TABLE routes ADD COLUMN reminder_hours INTEGER; + ALTER TABLE routes ADD COLUMN loud INTEGER NOT NULL DEFAULT 1; + ALTER TABLE routes ADD COLUMN use_default_targets INTEGER NOT NULL DEFAULT 0; + CREATE TABLE app_state ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + """, + """ + ALTER TABLE service_states ADD COLUMN ack_state INTEGER; + ALTER TABLE service_states ADD COLUMN host_held INTEGER NOT NULL DEFAULT 0; + INSERT OR IGNORE INTO app_state(key, value) + SELECT 'baseline.done', '1' WHERE EXISTS (SELECT 1 FROM service_states); + """, + """ + UPDATE routes SET reminder_hours = NULL WHERE reminder_hours = 0; + """, +] diff --git a/app/models.py b/app/models.py index c8d0c68..42a937e 100644 --- a/app/models.py +++ b/app/models.py @@ -4,15 +4,24 @@ KIND_PROBLEM = "problem" KIND_RECOVERY = "recovery" KIND_FLAPPING = "flapping" KIND_REMINDER = "reminder" +KIND_SYSTEM = "system" STATE_OK = 0 STATE_CRIT = 2 STATE_NAMES = {0: "OK", 1: "WARN", 2: "CRIT", 3: "UNKNOWN"} +HOST_SERVICE_NAME = "Host" +HOST_STATE_NAMES = {0: "UP", 2: "DOWN", 3: "UNREACH"} +# CRIT vor UNKNOWN vor WARN; passt wegen DOWN=2 und UNREACH=3 auch für Hosts. +SEVERITY_RANK = {0: 0, 1: 1, 3: 2, 2: 3} + SUPPRESS_DOWNTIME = "downtime" SUPPRESS_MUTE = "mute" SUPPRESS_RECOVERY_OFF = "recovery_aus" SUPPRESS_NOT_REPORTED = "nicht_gemeldet" +SUPPRESS_ACK = "bestaetigt" +SUPPRESS_HOST_DOWN = "host_down" +SUPPRESS_SEND_LIMIT = "sendelimit" AI_PENDING = "pending" AI_DONE = "done" @@ -26,6 +35,24 @@ def state_name(state: int) -> str: return STATE_NAMES.get(state, "UNKNOWN") +def is_host_service(service: str) -> bool: + return service == HOST_SERVICE_NAME + + +def display_state(service: str, state: int) -> str: + if is_host_service(service): + return HOST_STATE_NAMES.get(state, "DOWN") + return state_name(state) + + +def severity(state: int) -> int: + return SEVERITY_RANK.get(state, SEVERITY_RANK[3]) + + +def ack_holds(state: int, ack_state: int | None) -> bool: + return ack_state is not None and severity(state) <= severity(ack_state) + + @dataclass(frozen=True) class Event: ts: int @@ -63,6 +90,8 @@ class StateRow: reported_at: int = 0 reported_since: int = 0 last_reminder_ts: int = 0 + ack_state: int | None = None + host_held: bool = False @property def key(self) -> Key: diff --git a/app/poller/__init__.py b/app/poller/__init__.py index 8b3c8b6..d910819 100644 --- a/app/poller/__init__.py +++ b/app/poller/__init__.py @@ -1,8 +1,12 @@ +from .baseline import BaselineGuard, baseline_rows, open_rows from .events import compute_events from .loop import Poller -from .options import PollOptions, read_poll_options +from .notify import SystemNotifier +from .options import PollOptions, read_host_delay, read_poll_options +from .outage import FAILURE_THRESHOLD, OutageNotice, OutageTracker, RestoredNotice, classify from .state import StateChange, StateStore from .status import PollerStatus -__all__ = ["PollOptions", "Poller", "PollerStatus", "StateChange", "StateStore", "compute_events", - "read_poll_options"] +__all__ = ["BaselineGuard", "FAILURE_THRESHOLD", "OutageNotice", "OutageTracker", "PollOptions", "Poller", + "PollerStatus", "RestoredNotice", "StateChange", "StateStore", "SystemNotifier", "baseline_rows", + "classify", "compute_events", "open_rows", "read_host_delay", "read_poll_options"] diff --git a/app/poller/baseline.py b/app/poller/baseline.py new file mode 100644 index 0000000..6b8dd1b --- /dev/null +++ b/app/poller/baseline.py @@ -0,0 +1,38 @@ +from typing import Mapping, Sequence + +from ..checkmk import ServiceState +from ..db import Database +from ..models import STATE_OK, Key, StateRow +from .events import build_row + +BASELINE_DONE_KEY = "baseline.done" +BASELINE_REQUEST_KEY = "baseline.requested" +FLAG = "1" + + +def baseline_rows(previous: Mapping[Key, StateRow], current: Sequence[ServiceState], now: int) -> list[StateRow]: + return [build_row(cur, previous.get((cur.host, cur.service)), now, cur.state, cur.flapping, + (now, cur.last_state_change)) + for cur in current if cur.hard] + + +def open_rows(rows: Sequence[StateRow]) -> list[StateRow]: + return [row for row in rows if row.state != STATE_OK] + + +class BaselineGuard: + def __init__(self, db: Database): + self._db = db + + def _is_set(self, key: str) -> bool: + return bool(self._db.query("SELECT 1 FROM app_state WHERE key=?", (key,))) + + def needed(self, state_empty: bool) -> bool: + return self._is_set(BASELINE_REQUEST_KEY) or (state_empty and not self._is_set(BASELINE_DONE_KEY)) + + def request(self) -> None: + self._db.execute("INSERT OR REPLACE INTO app_state(key, value) VALUES (?, ?)", (BASELINE_REQUEST_KEY, FLAG)) + + def completion(self) -> list[tuple[str, tuple]]: + return [("INSERT OR REPLACE INTO app_state(key, value) VALUES (?, ?)", (BASELINE_DONE_KEY, FLAG)), + ("DELETE FROM app_state WHERE key=?", (BASELINE_REQUEST_KEY,))] diff --git a/app/poller/events.py b/app/poller/events.py index 36d877c..9aac009 100644 --- a/app/poller/events.py +++ b/app/poller/events.py @@ -2,8 +2,8 @@ from dataclasses import replace from typing import Callable, Mapping, Sequence from ..checkmk import ServiceState -from ..models import (KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, STATE_OK, SUPPRESS_DOWNTIME, Event, Key, - StateRow) +from ..models import (HOST_SERVICE_NAME, KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, STATE_OK, SUPPRESS_ACK, + SUPPRESS_DOWNTIME, SUPPRESS_HOST_DOWN, Event, Key, StateRow, ack_holds) FLAP_QUIET_SECONDS = 600 @@ -20,41 +20,67 @@ def _problem_since(cur: ServiceState, prev: StateRow | None, now: int) -> int: return cur.last_state_change or now -def _row(cur: ServiceState, prev: StateRow | None, now: int, reported_state: int, flap_reported: bool, - reported: tuple[int, int] | None = None) -> StateRow: +def _ack_state(cur: ServiceState, prev: StateRow | None) -> int | None: + if not cur.acknowledged: + return None + return prev.ack_state if prev is not None and prev.ack_state is not None else cur.state + + +def build_row(cur: ServiceState, prev: StateRow | None, now: int, reported_state: int, flap_reported: bool, + reported: tuple[int, int] | None = None, held: bool = False) -> StateRow: reported_at, reported_since = reported or ((prev.reported_at, prev.reported_since) if prev else (0, 0)) return StateRow(cur.host, cur.service, cur.state, cur.output, cur.last_state_change, cur.in_downtime, cur.flapping, reported_state, flap_reported, _problem_since(cur, prev, now), - reported_at, reported_since, prev.last_reminder_ts if prev else 0) + reported_at, reported_since, prev.last_reminder_ts if prev else 0, + _ack_state(cur, prev), held) -def _event(now: int, kind: str, cur: ServiceState, row: StateRow, **extra) -> Event: +def _reason(cur: ServiceState, ack_state: int | None, down_hosts: frozenset[str]) -> str | None: + if cur.in_downtime: + return SUPPRESS_DOWNTIME + if cur.service != HOST_SERVICE_NAME and cur.host in down_hosts: + return SUPPRESS_HOST_DOWN + if ack_holds(cur.state, ack_state): + return SUPPRESS_ACK + return None + + +def _event(now: int, kind: str, cur: ServiceState, row: StateRow, reason: str | None, **extra) -> Event: return Event(ts=now, host=cur.host, service=cur.service, kind=kind, state=cur.state, output=cur.output, - suppressed=cur.in_downtime, suppress_reason=SUPPRESS_DOWNTIME if cur.in_downtime else None, + suppressed=reason is not None, suppress_reason=reason, since_ts=cur.last_state_change, problem_since_ts=row.problem_since, **extra) -def _problem_event(now: int, cur: ServiceState, prev: StateRow | None, row: StateRow) -> Event: +def _problem_event(now: int, cur: ServiceState, prev: StateRow | None, row: StateRow, + reason: str | None) -> Event: if prev is None or prev.reported_state == STATE_OK: - return _event(now, KIND_PROBLEM, cur, row) + return _event(now, KIND_PROBLEM, cur, row, reason) duration = max(cur.last_state_change - prev.reported_since, 0) if prev.reported_since else None - return _event(now, KIND_PROBLEM, cur, row, previous_state=prev.reported_state, previous_duration=duration) + return _event(now, KIND_PROBLEM, cur, row, reason, previous_state=prev.reported_state, + previous_duration=duration) -def _handle_current(cur: ServiceState, prev: StateRow | None, now: int, - delay_seconds: DelayProvider) -> tuple[StateRow | None, Event | None]: +def _handle_current(cur: ServiceState, prev: StateRow | None, now: int, delay_seconds: DelayProvider, + down_hosts: frozenset[str]) -> tuple[StateRow | None, Event | None]: if not cur.hard: return prev, None reported = prev.reported_state if prev else STATE_OK flap_reported = prev.flap_reported if prev else False + ack = _ack_state(cur, prev) + reason = _reason(cur, ack, down_hosts) if cur.flapping: - row = _row(cur, prev, now, reported, True) - return row, (None if flap_reported else _event(now, KIND_FLAPPING, cur, row)) + row = build_row(cur, prev, now, reported, True) + return row, (None if flap_reported else _event(now, KIND_FLAPPING, cur, row, reason)) delay = delay_seconds(cur.host, cur.service, cur.state, cur.output) if now - cur.last_state_change < delay or cur.state == reported: - return _row(cur, prev, now, reported, False), None - row = _row(cur, prev, now, cur.state, False, (now, cur.last_state_change)) - return row, _problem_event(now, cur, prev, row) + return build_row(cur, prev, now, reported, False), None + if reason == SUPPRESS_HOST_DOWN: + # Nicht als gemeldet markieren: bleibt das Problem nach der Rückkehr des Hosts bestehen, wird es normal gemeldet. + already = prev is not None and prev.host_held and prev.state == cur.state + row = build_row(cur, prev, now, reported, False, held=True) + return row, (None if already else _problem_event(now, cur, prev, row, reason)) + row = build_row(cur, prev, now, cur.state, False, (now, cur.last_state_change)) + return row, _problem_event(now, cur, prev, row, reason) def _handle_gone(prev: StateRow, now: int) -> tuple[StateRow | None, Event | None]: @@ -64,7 +90,7 @@ def _handle_gone(prev: StateRow, now: int) -> tuple[StateRow | None, Event | Non if prev.state == STATE_OK: return prev, None held = replace(prev, state=STATE_OK, output="", last_state_change=now, in_downtime=False, - flapping=True, flap_reported=True) + flapping=True, flap_reported=True, ack_state=None, host_held=False) return held, None if prev.reported_state == STATE_OK: return None, None @@ -75,15 +101,20 @@ def _handle_gone(prev: StateRow, now: int) -> tuple[StateRow | None, Event | Non return None, event +def hosts_not_up(current: Sequence[ServiceState]) -> frozenset[str]: + return frozenset(c.host for c in current if c.service == HOST_SERVICE_NAME and c.state != STATE_OK) + + def compute_events(previous: Mapping[Key, StateRow], current: Sequence[ServiceState], now: int, delay_seconds: DelayProvider = no_delay) -> tuple[list[Event], list[StateRow]]: events: list[Event] = [] rows: dict[Key, StateRow] = {} seen: set[Key] = set() + down_hosts = hosts_not_up(current) for cur in current: key = (cur.host, cur.service) seen.add(key) - row, event = _handle_current(cur, previous.get(key), now, delay_seconds) + row, event = _handle_current(cur, previous.get(key), now, delay_seconds, down_hosts) if row is not None: rows[key] = row if event is not None: diff --git a/app/poller/loop.py b/app/poller/loop.py index 4ecfdcf..3bb6a0f 100644 --- a/app/poller/loop.py +++ b/app/poller/loop.py @@ -3,11 +3,14 @@ import logging import time from typing import Awaitable, Callable -from ..checkmk import CheckmkClient +from ..checkmk import CheckmkClient, ServiceState from ..models import Event from ..routing import Route, delay_provider +from .baseline import BaselineGuard, baseline_rows, open_rows from .events import compute_events +from .notify import SystemNotifier from .options import PollOptions, read_poll_options +from .outage import OutageTracker, classify from .state import StateStore from .status import PollerStatus @@ -22,7 +25,9 @@ RuleSource = Callable[[], list[Route]] class Poller: def __init__(self, store, state: StateStore, sink: EventSink, status: PollerStatus, client_factory: Callable[[], CheckmkClient | None] | None = None, - clock: Callable[[], float] = time.time, rules: RuleSource | None = None): + clock: Callable[[], float] = time.time, rules: RuleSource | None = None, + notifier: SystemNotifier | None = None, baseline: BaselineGuard | None = None, + tracker: OutageTracker | None = None): self._store = store self._state = state self._sink = sink @@ -30,6 +35,9 @@ class Poller: self._client_factory = client_factory or (lambda: CheckmkClient.from_settings(store)) self._clock = clock self._rules = rules or (lambda: []) + self._notifier = notifier + self._baseline = baseline + self._tracker = tracker or OutageTracker() async def run(self) -> None: while True: @@ -49,23 +57,61 @@ class Poller: if client is None: return try: - current = await client.problem_services() + current = await self._fetch(client) except Exception as exc: - self.status.last_error = f"{type(exc).__name__}: {exc}"[:ERROR_MAX] - log.warning("Checkmk-Abfrage fehlgeschlagen: %s", self.status.last_error) + await self._on_failure(exc) return finally: await client.aclose() now = int(self._clock()) + await self._on_reachable(now) previous = await asyncio.to_thread(self._state.load) + if self._baseline is not None and await asyncio.to_thread(self._baseline.needed, not previous): + await self._record_baseline(previous, current, now) + return rules = await asyncio.to_thread(self._rules) - delay = delay_provider(rules, options.default_delay_minutes) + delay = delay_provider(rules, options.default_delay_minutes, options.host_delay_minutes) events, rows = compute_events(previous, current, now, delay) await asyncio.to_thread(self._state.apply, previous, rows, now) - self.status.last_success = self._clock() - self.status.last_error = None + self._mark_success() if events: try: await self._sink(events) except Exception: log.exception("Verarbeitung der Events fehlgeschlagen") + + async def _fetch(self, client: CheckmkClient) -> list[ServiceState]: + services = await client.problem_services() + hosts = await client.problem_hosts() + return [*services, *(host.as_service() for host in hosts)] + + def _mark_success(self) -> None: + self.status.last_success = self._clock() + self.status.last_error = None + + async def _record_baseline(self, previous, current: list[ServiceState], now: int) -> None: + rows = baseline_rows(previous, current, now) + await asyncio.to_thread(self._state.apply, previous, rows, now, self._baseline.completion()) + self._mark_success() + if self._notifier is not None: + await self._notify(self._notifier.baseline(open_rows(rows))) + + async def _on_failure(self, exc: Exception) -> None: + self.status.last_error = f"{type(exc).__name__}: {exc}"[:ERROR_MAX] + log.warning("Checkmk-Abfrage fehlgeschlagen: %s", self.status.last_error) + kind, brief = classify(exc) + notice = self._tracker.failed(kind, brief, int(self._clock())) + if notice is not None and self._notifier is not None: + await self._notify(self._notifier.outage(notice)) + + async def _on_reachable(self, now: int) -> None: + notice = self._tracker.succeeded(now) + if notice is not None and self._notifier is not None: + await self._notify(self._notifier.restored(notice)) + + @staticmethod + async def _notify(call: Awaitable[None]) -> None: + try: + await call + except Exception: + log.exception("Systemmeldung konnte nicht gesendet werden") diff --git a/app/poller/notify.py b/app/poller/notify.py new file mode 100644 index 0000000..580df24 --- /dev/null +++ b/app/poller/notify.py @@ -0,0 +1,12 @@ +from typing import Protocol, Sequence + +from ..models import StateRow +from .outage import OutageNotice, RestoredNotice + + +class SystemNotifier(Protocol): + async def outage(self, notice: OutageNotice) -> None: ... + + async def restored(self, notice: RestoredNotice) -> None: ... + + async def baseline(self, rows: Sequence[StateRow]) -> None: ... diff --git a/app/poller/options.py b/app/poller/options.py index 17db1e7..357f3a5 100644 --- a/app/poller/options.py +++ b/app/poller/options.py @@ -1,21 +1,28 @@ from dataclasses import dataclass -from ..constants import (ALERTS_DEFAULT_DELAY_MINUTES, DEFAULT_DELAY_MINUTES, DEFAULT_POLL_INTERVAL, - MAX_DELAY_MINUTES, MIN_POLL_INTERVAL, POLL_INTERVAL) +from ..constants import (ALERTS_DEFAULT_DELAY_MINUTES, ALERTS_HOST_DELAY_MINUTES, DEFAULT_DELAY_MINUTES, + DEFAULT_HOST_DELAY_MINUTES, DEFAULT_POLL_INTERVAL, MAX_DELAY_MINUTES, MIN_POLL_INTERVAL, + POLL_INTERVAL) @dataclass(frozen=True) class PollOptions: interval: int default_delay_minutes: int + host_delay_minutes: int = DEFAULT_HOST_DELAY_MINUTES def read_default_delay(store) -> int: return min(store.get_int(ALERTS_DEFAULT_DELAY_MINUTES, DEFAULT_DELAY_MINUTES), MAX_DELAY_MINUTES) +def read_host_delay(store) -> int: + return min(store.get_int(ALERTS_HOST_DELAY_MINUTES, DEFAULT_HOST_DELAY_MINUTES), MAX_DELAY_MINUTES) + + def read_poll_options(store) -> PollOptions: return PollOptions( interval=store.get_int(POLL_INTERVAL, DEFAULT_POLL_INTERVAL, MIN_POLL_INTERVAL), default_delay_minutes=read_default_delay(store), + host_delay_minutes=read_host_delay(store), ) diff --git a/app/poller/outage.py b/app/poller/outage.py new file mode 100644 index 0000000..d64d51a --- /dev/null +++ b/app/poller/outage.py @@ -0,0 +1,47 @@ +from dataclasses import dataclass + +from ..checkmk import FAILURE_UNEXPECTED, CheckmkError + +FAILURE_THRESHOLD = 3 + + +@dataclass(frozen=True) +class OutageNotice: + kind: str + since: int + brief: str + + +@dataclass(frozen=True) +class RestoredNotice: + since: int + now: int + + +def classify(exc: Exception) -> tuple[str, str]: + if isinstance(exc, CheckmkError): + return exc.kind, exc.brief + return FAILURE_UNEXPECTED, type(exc).__name__ + + +class OutageTracker: + def __init__(self, threshold: int = FAILURE_THRESHOLD): + self._threshold = threshold + self._failures = 0 + self._since = 0 + self._announced = False + + def failed(self, kind: str, brief: str, now: int) -> OutageNotice | None: + if self._failures == 0: + self._since = now + self._failures += 1 + if self._failures < self._threshold or self._announced: + return None + self._announced = True + return OutageNotice(kind, self._since, brief) + + def succeeded(self, now: int) -> RestoredNotice | None: + notice = RestoredNotice(self._since, now) if self._announced else None + self._failures = 0 + self._announced = False + return notice diff --git a/app/poller/state.py b/app/poller/state.py index e4ebb64..c7c7ef4 100644 --- a/app/poller/state.py +++ b/app/poller/state.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from typing import Sequence from ..db import Database from ..models import STATE_OK, Key, StateRow @@ -9,12 +10,13 @@ RECENT_CHANGES_DEFAULT = 5 UPSERT_SQL = ( "INSERT INTO service_states(host, service, state, output, last_state_change, in_downtime, " "flapping, reported_state, flap_reported, problem_since, reported_at, reported_since, " - "last_reminder_ts) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) " + "last_reminder_ts, ack_state, host_held) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) " "ON CONFLICT(host, service) DO UPDATE SET state=excluded.state, output=excluded.output, " "last_state_change=excluded.last_state_change, in_downtime=excluded.in_downtime, " "flapping=excluded.flapping, reported_state=excluded.reported_state, " "flap_reported=excluded.flap_reported, problem_since=excluded.problem_since, " - "reported_at=excluded.reported_at, reported_since=excluded.reported_since" + "reported_at=excluded.reported_at, reported_since=excluded.reported_since, " + "ack_state=excluded.ack_state, host_held=excluded.host_held" ) LOG_SQL = "INSERT INTO state_log(host, service, state, ts) VALUES (?,?,?,?)" @@ -29,7 +31,7 @@ def _from_db(row) -> StateRow: return StateRow(row["host"], row["service"], row["state"], row["output"], row["last_state_change"], bool(row["in_downtime"]), bool(row["flapping"]), row["reported_state"], bool(row["flap_reported"]), row["problem_since"], row["reported_at"], - row["reported_since"], row["last_reminder_ts"]) + row["reported_since"], row["last_reminder_ts"], row["ack_state"], bool(row["host_held"])) class StateStore: @@ -40,7 +42,8 @@ class StateStore: rows = self._db.query("SELECT * FROM service_states") return {(r["host"], r["service"]): _from_db(r) for r in rows} - def apply(self, previous: dict[Key, StateRow], rows: list[StateRow], now: int) -> None: + def apply(self, previous: dict[Key, StateRow], rows: list[StateRow], now: int, + extra: Sequence[tuple[str, tuple]] = ()) -> None: statements: list[tuple[str, tuple]] = [] new_keys = {row.key for row in rows} for key, prev in previous.items(): @@ -58,7 +61,9 @@ class StateStore: statements.append((UPSERT_SQL, (row.host, row.service, row.state, row.output, row.last_state_change, int(row.in_downtime), int(row.flapping), row.reported_state, int(row.flap_reported), row.problem_since, - row.reported_at, row.reported_since, row.last_reminder_ts))) + row.reported_at, row.reported_since, row.last_reminder_ts, + row.ack_state, int(row.host_held)))) + statements.extend(extra) if statements: self._db.execute_many(statements) diff --git a/app/reminders/due.py b/app/reminders/due.py index 0812efa..40dfd41 100644 --- a/app/reminders/due.py +++ b/app/reminders/due.py @@ -1,8 +1,8 @@ from typing import Iterable, Sequence -from ..models import KIND_PROBLEM, KIND_REMINDER, STATE_CRIT, Event, StateRow +from ..models import KIND_PROBLEM, KIND_REMINDER, STATE_CRIT, Event, StateRow, ack_holds from ..mutes import Mute, find_active_mute -from ..routing import Route, reminder_interval +from ..routing import Route, resolve_reminder_hours SECONDS_PER_HOUR = 3600 @@ -13,10 +13,12 @@ def due_reminders(rows: Iterable[StateRow], rules: Sequence[Route], mutes: Seque for row in rows: if row.state != STATE_CRIT or row.reported_state != STATE_CRIT or row.in_downtime or row.flapping: continue + if ack_holds(row.state, row.ack_state): + continue if find_active_mute(mutes, row.host, row.service, now): continue probe = Event(now, row.host, row.service, KIND_PROBLEM, row.state, row.output) - hours = reminder_interval(rules, probe) or default_hours + hours = resolve_reminder_hours(rules, probe, default_hours) if hours <= 0 or now - max(row.last_reminder_ts, row.reported_at) < hours * SECONDS_PER_HOUR: continue events.append(Event(now, row.host, row.service, KIND_REMINDER, row.state, row.output, diff --git a/app/routing/__init__.py b/app/routing/__init__.py index ff3083c..81d4dd1 100644 --- a/app/routing/__init__.py +++ b/app/routing/__init__.py @@ -1,4 +1,4 @@ -from .decisions import ai_allowed, ai_hints, delay_provider, reminder_interval, resolve_delay, resolve_loud +from .decisions import ai_allowed, ai_hints, delay_provider, resolve_delay, resolve_loud, resolve_reminder_hours from .limits import DEFAULT_MIN_STATE from .repository import RouteRepository from .rules import ALL_KINDS, Route, matches, matching_rules, resolve_targets, route_matches @@ -9,6 +9,6 @@ from .validation import BACKENDS, RouteInput, validate_route __all__ = [ "ALL_KINDS", "BACKENDS", "DEFAULT_MIN_STATE", "DIRECTION_DOWN", "DIRECTION_UP", "Route", "RouteError", "RouteInput", "RouteRepository", "RouteService", "ZPOOL_MARKER", "ai_allowed", "ai_hints", - "delay_provider", "matches", "matching_rules", "reminder_interval", "resolve_delay", "resolve_loud", + "delay_provider", "matches", "matching_rules", "resolve_delay", "resolve_loud", "resolve_reminder_hours", "resolve_targets", "route_matches", "seed_default_rules", "validate_route", "zpool_rule", ] diff --git a/app/routing/decisions.py b/app/routing/decisions.py index 943aafb..7eaacba 100644 --- a/app/routing/decisions.py +++ b/app/routing/decisions.py @@ -1,6 +1,6 @@ from typing import Callable, Sequence -from ..models import KIND_PROBLEM, Event +from ..models import KIND_PROBLEM, Event, is_host_service from .limits import AI_HINTS_TOTAL_MAX from .rules import Route, matching_rules @@ -18,8 +18,11 @@ def resolve_delay(rules: Sequence[Route], host: str, service: str, state: int, o return min(delays) if delays else default_minutes -def delay_provider(rules: Sequence[Route], default_minutes: int) -> Callable[[str, str, int, str], int]: +def delay_provider(rules: Sequence[Route], default_minutes: int, + host_minutes: int = 0) -> Callable[[str, str, int, str], int]: def seconds(host: str, service: str, state: int, output: str) -> int: + if is_host_service(service): + return host_minutes * SECONDS_PER_MINUTE return resolve_delay(rules, host, service, state, output, default_minutes) * SECONDS_PER_MINUTE return seconds @@ -38,6 +41,8 @@ def ai_hints(rules: Sequence[Route], event: Event) -> str: return "\n".join(hints)[:AI_HINTS_TOTAL_MAX] -def reminder_interval(rules: Sequence[Route], event: Event) -> int: - hours = [r.reminder_hours for r in matching_rules(rules, event) if r.reminder_hours and r.reminder_hours > 0] - return min(hours) if hours else 0 +def resolve_reminder_hours(rules: Sequence[Route], event: Event, default_hours: int) -> int: + values = [r.reminder_hours for r in matching_rules(rules, event) if r.reminder_hours is not None] + if any(value <= 0 for value in values): + return 0 + return min(values) if values else default_hours diff --git a/app/routing/repository.py b/app/routing/repository.py index e067d1f..3e6e181 100644 --- a/app/routing/repository.py +++ b/app/routing/repository.py @@ -36,7 +36,7 @@ def _values(data: RouteInput) -> tuple: return (data.name.strip(), int(data.enabled), data.match_host.strip(), data.match_service.strip(), data.min_state, KINDS_SEP.join(data.kinds), int(data.stop_on_match), targets, data.delay_minutes, data.output_contains.strip(), int(data.ai_enabled), data.ai_hint.strip(), - data.reminder_hours or None, int(data.loud), int(data.use_default_targets)) + data.reminder_hours, int(data.loud), int(data.use_default_targets)) def insert_statement(data: RouteInput) -> tuple[str, tuple]: diff --git a/app/routing/service.py b/app/routing/service.py index ce48ad4..c45a935 100644 --- a/app/routing/service.py +++ b/app/routing/service.py @@ -18,11 +18,12 @@ def describe(route: Route | RouteInput) -> str: targets = ", ".join(f"{t.backend}:{t.target}" for t in route.targets) or "keine" state = "an" if route.enabled else "aus" delay = "std" if route.delay_minutes is None else route.delay_minutes + reminder = "std" if route.reminder_hours is None else route.reminder_hours return (f"{route.name} [{state}] host={route.match_host or '*'} service={route.match_service or '*'} " f"min_state={route.min_state} kinds={','.join(route.kinds)} " f"stop={int(route.stop_on_match)} delay={delay} contains={route.output_contains!r} " f"ai={int(route.ai_enabled)} hint_len={len(route.ai_hint)} " - f"reminder_h={route.reminder_hours or 0} loud={int(route.loud)} " + f"reminder_h={reminder} loud={int(route.loud)} " f"default_targets={int(route.use_default_targets)} -> {targets}") diff --git a/app/runtime.py b/app/runtime.py index a28d285..304ecb4 100644 --- a/app/runtime.py +++ b/app/runtime.py @@ -5,7 +5,9 @@ from contextlib import asynccontextmanager from fastapi import FastAPI from . import audit -from .alerts import AlertActions, AlertAnalyzer, AlertPipeline, AlertRepository, SlidingWindowLimiter +from .alerts import (AlertActions, AlertAnalyzer, AlertPipeline, AlertRepository, SendLimiter, + SlidingWindowLimiter, SystemNotifier, base_url_provider, content_mode_provider, + send_limit_provider) from .chat import AdapterRegistry from .chatops import ChatOps from .constants import (ALERTS_BUNDLE_SECONDS, ALERTS_RECOVERY_ENABLED, ALERTS_REMINDER_HOURS, DEFAULT_BUNDLE_SECONDS, @@ -14,7 +16,7 @@ from .db import Database from .agent_runs import configured_secrets from .mutes import MuteService from .ollama import OllamaClient -from .poller import Poller, PollerStatus, StateStore +from .poller import BaselineGuard, Poller, PollerStatus, StateStore from .prompts import (PROMPT_AGENT, PROMPT_ALERT, PromptRepository, PromptService, PromptTester, UserRateLimiter, default_registry) from .retention import RetentionRepository, RetentionRunner, RetentionService, policy_from @@ -38,14 +40,20 @@ class Runtime: analyzer = AlertAnalyzer(store, lambda: OllamaClient.from_settings(store), self.state.recent_changes, SlidingWindowLimiter(time.time), lambda: self.prompts.effective_text(PROMPT_ALERT)) + actions, limiter, send_limit = AlertActions(store), SendLimiter(time.time), send_limit_provider(store) self.pipeline = AlertPipeline( self.alerts, self.registry.current, analyzer, lambda: store.get_int(ALERTS_BUNDLE_SECONDS, DEFAULT_BUNDLE_SECONDS), - routes=self.routes, mutes=self.mutes, actions=AlertActions(store), + routes=self.routes, mutes=self.mutes, actions=actions, recovery_enabled=lambda: store.get(ALERTS_RECOVERY_ENABLED, FLAG_ON) != FLAG_OFF, + limiter=limiter, send_limit=send_limit, content_mode=content_mode_provider(store), + base_url=base_url_provider(store), ) + self.baseline = BaselineGuard(db) + self.notifier = SystemNotifier(self.alerts, self.registry.current, limiter, send_limit, actions, + on_baseline=self.pipeline.reset_reported) self.poller = Poller(store, self.state, self.pipeline.handle, self.status, - rules=self.routes.enabled_routes) + rules=self.routes.enabled_routes, notifier=self.notifier, baseline=self.baseline) self.reminders = ReminderWorker( self.state.load, self.routes.enabled_routes, self.mutes.active, lambda: min(store.get_int(ALERTS_REMINDER_HOURS, DEFAULT_REMINDER_HOURS), MAX_REMINDER_HOURS), diff --git a/app/web/alerting_section.py b/app/web/alerting_section.py index 006b14d..8eaf50b 100644 --- a/app/web/alerting_section.py +++ b/app/web/alerting_section.py @@ -1,11 +1,17 @@ +import re + from ..constants import (ALERTS_AI_PER_MINUTE, ALERTS_BUNDLE_SECONDS, ALERTS_DEFAULT_DELAY_MINUTES, - ALERTS_RECOVERY_ENABLED, ALERTS_REMINDER_HOURS, DEFAULT_AI_PER_MINUTE, - DEFAULT_BUNDLE_SECONDS, DEFAULT_DELAY_MINUTES, DEFAULT_POLL_INTERVAL, - DEFAULT_REMINDER_HOURS, MAX_DELAY_MINUTES, MAX_REMINDER_HOURS, MIN_POLL_INTERVAL, - POLL_INTERVAL) -from .sections import Field, Section + ALERTS_HOST_DELAY_MINUTES, ALERTS_RECOVERY_ENABLED, ALERTS_REMINDER_HOURS, + ALERTS_SEND_LIMIT, DEFAULT_AI_PER_MINUTE, DEFAULT_BUNDLE_SECONDS, + DEFAULT_DELAY_MINUTES, DEFAULT_HOST_DELAY_MINUTES, DEFAULT_POLL_INTERVAL, + DEFAULT_REMINDER_HOURS, DEFAULT_SEND_LIMIT, MAX_DELAY_MINUTES, MAX_REMINDER_HOURS, + MAX_SEND_LIMIT, MIN_POLL_INTERVAL, MIN_SEND_LIMIT, POLL_INTERVAL, UI_BASE_URL) +from .sections import Action, Field, Section SLUG = "alerting" +BASELINE_PATH = f"/settings/{SLUG}/baseline" +BASE_URL_MAX = 200 +BASE_URL_RE = re.compile(r"^https?://[^\s/?#]+(/[^\s?#]*)?$") MAX_POLL_INTERVAL = 3600 MAX_BUNDLE_SECONDS = 3600 @@ -17,9 +23,15 @@ LIMITS = ( (ALERTS_REMINDER_HOURS, "Erinnerung", 0, MAX_REMINDER_HOURS), (ALERTS_BUNDLE_SECONDS, "Bündelungsfenster", 0, MAX_BUNDLE_SECONDS), (ALERTS_AI_PER_MINUTE, "KI-Analysen pro Minute", 0, MAX_AI_PER_MINUTE), + (ALERTS_HOST_DELAY_MINUTES, "Host-Verzögerung", 0, MAX_DELAY_MINUTES), + (ALERTS_SEND_LIMIT, "Sendelimit", MIN_SEND_LIMIT, MAX_SEND_LIMIT), ) +def strip_trailing_slash(value: str) -> str: + return value.rstrip("/") + + def validate(values: dict) -> list[str]: errors = [] for key, label, low, high in LIMITS: @@ -28,6 +40,10 @@ def validate(values: dict) -> list[str]: continue if not raw.isdecimal() or not low <= int(raw) <= high: errors.append(f"{label} muss eine ganze Zahl zwischen {low} und {high} sein.") + base_url = values.get(UI_BASE_URL, "").strip() + if base_url and (len(base_url) > BASE_URL_MAX or not BASE_URL_RE.match(base_url.rstrip("/"))): + errors.append("Basis-URL der Oberfläche muss mit http:// oder https:// beginnen und darf keine " + "Leerzeichen, Parameter oder Anker enthalten.") return errors @@ -47,7 +63,20 @@ SECTION = Section( default=str(DEFAULT_BUNDLE_SECONDS)), Field(ALERTS_AI_PER_MINUTE, "Max. KI-Analysen pro Minute (0 = keine)", default=str(DEFAULT_AI_PER_MINUTE)), + Field(ALERTS_HOST_DELAY_MINUTES, + "Host-Verzögerung in Minuten (Host DOWN/UNREACH wird erst nach dieser Zeit gemeldet, 0 = sofort)", + default=str(DEFAULT_HOST_DELAY_MINUTES), + hint="Regel-Verzögerungen gelten nicht für Host-Alarme. Solange ein Host nicht UP ist, werden " + "Service-Alarme dieses Hosts unterdrückt."), + Field(ALERTS_SEND_LIMIT, "Sendelimit: Nachrichten pro Minute und Ziel (1 bis 60)", + default=str(DEFAULT_SEND_LIMIT), + hint="Überschuss wird nicht einzeln gesendet, sondern als eine Sammelmeldung pro Minute und Ziel; " + "alle Alarme bleiben im Verlauf."), + Field(UI_BASE_URL, "Basis-URL dieser Oberfläche (optional)", placeholder="https://checkbot.example.de", + transform=strip_trailing_slash, + hint="Wenn gesetzt, enthält jede Alarmnachricht einen Link \"Details\" auf den Verlauf."), ), validate=validate, optional=True, + actions=(Action("Baseline neu setzen", BASELINE_PATH),), ) diff --git a/app/web/alerts.py b/app/web/alerts.py index 92e607b..96f426c 100644 --- a/app/web/alerts.py +++ b/app/web/alerts.py @@ -6,7 +6,8 @@ from .deps import current_user, render router = APIRouter() SUPPRESS_LABELS = {"mute": "Stummschaltung", "downtime": "Downtime", "recovery_aus": "Recovery aus", - "nicht_gemeldet": "Problem war nicht gemeldet"} + "nicht_gemeldet": "Problem war nicht gemeldet", "bestaetigt": "bestätigt", + "host_down": "Host down", "sendelimit": "im Sammelhinweis"} AI_LABELS = {"pending": "läuft", "done": "fertig", "failed": "nicht verfügbar", "skipped": "übersprungen"} diff --git a/app/web/baseline_pages.py b/app/web/baseline_pages.py new file mode 100644 index 0000000..29be42d --- /dev/null +++ b/app/web/baseline_pages.py @@ -0,0 +1,22 @@ +import asyncio + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import JSONResponse + +from .. import audit +from ..users import User +from .alerting_section import BASELINE_PATH +from .deps import require_admin + +router = APIRouter() + +AUDIT_ACTION = "baseline.request" +MSG_REQUESTED = ("Baseline angefordert: Der nächste erfolgreiche Abruf zeichnet den Zustand still auf und " + "sendet eine Zusammenfassung.") + + +@router.post(BASELINE_PATH) +async def request_baseline(request: Request, admin: User = Depends(require_admin)): + await asyncio.to_thread(request.app.state.runtime.baseline.request) + await asyncio.to_thread(audit.record, request.app.state.db, admin.username, AUDIT_ACTION, "baseline") + return JSONResponse({"ok": True, "detail": MSG_REQUESTED}) diff --git a/app/web/content_mode.py b/app/web/content_mode.py new file mode 100644 index 0000000..575d1a4 --- /dev/null +++ b/app/web/content_mode.py @@ -0,0 +1,19 @@ +from ..constants import CONTENT_MODE_FULL, CONTENT_MODE_SHORT, CONTENT_MODES +from .sections import Field + +CHOICES = ( + (CONTENT_MODE_FULL, "Voll (mit Plugin-Output und KI-Analyse)"), + (CONTENT_MODE_SHORT, "Kurz (nur Host, Service, Status und Verlauf, ohne Output und KI-Text)"), +) + + +def content_mode_field(key: str, hint: str) -> Field: + return Field(key, "Inhalt der Alarmnachrichten", kind="select", default=CONTENT_MODE_FULL, + choices=CHOICES, hint=hint) + + +def validate_content_mode(values: dict, key: str) -> list[str]: + value = values.get(key, "") + if value and value not in CONTENT_MODES: + return ["Inhalt der Alarmnachrichten muss \"voll\" oder \"kurz\" sein."] + return [] diff --git a/app/web/deps.py b/app/web/deps.py index 13dc56f..71e7548 100644 --- a/app/web/deps.py +++ b/app/web/deps.py @@ -7,7 +7,7 @@ from fastapi.responses import RedirectResponse from fastapi.templating import Jinja2Templates from markupsafe import Markup, escape -from ..models import state_name +from ..models import display_state, state_name from ..users import User BASE_DIR = Path(__file__).parent @@ -32,6 +32,7 @@ templates.env.filters["fmt_ts"] = lambda ts: datetime.fromtimestamp(ts).astimezo STATE_CLASSES = {0: "ok", 1: "warn", 2: "error", 3: "error"} templates.env.filters["state_name"] = state_name +templates.env.filters["display_state"] = lambda state, service: display_state(service, state) templates.env.filters["state_class"] = lambda state: STATE_CLASSES.get(state, "error") diff --git a/app/web/matrix_section.py b/app/web/matrix_section.py index a31cf89..b37ee47 100644 --- a/app/web/matrix_section.py +++ b/app/web/matrix_section.py @@ -5,9 +5,10 @@ from fastapi.responses import JSONResponse from .. import audit from ..chat import MATRIX_ROOM_ID_RE, MATRIX_USER_ID_RE, MatrixAdapter, mask_secret -from ..constants import (FLAG_OFF, FLAG_ON, MATRIX_ALLOWLIST, MATRIX_ENABLED, MATRIX_HOMESERVER, - MATRIX_ROOM_ID, MATRIX_TOKEN, MATRIX_USER_ID) +from ..constants import (FLAG_OFF, FLAG_ON, MATRIX_ALLOWLIST, MATRIX_CONTENT_MODE, MATRIX_ENABLED, + MATRIX_HOMESERVER, MATRIX_ROOM_ID, MATRIX_TOKEN, MATRIX_USER_ID) from ..users import User +from .content_mode import content_mode_field, validate_content_mode from .deps import require_admin from .sections import URL_RE, Action, Field, Section, check, collect, effective @@ -43,7 +44,7 @@ def validate(values: dict) -> list[str]: entries = [e.strip() for e in allowlist.split(",")] if not all(MATRIX_USER_ID_RE.match(e) for e in entries): errors.append("Allowlist darf nur Matrix-IDs (@name:server), durch Kommas getrennt, enthalten.") - return errors + return errors + validate_content_mode(values, MATRIX_CONTENT_MODE) def _adapter(values: dict) -> MatrixAdapter: @@ -73,6 +74,9 @@ SECTION = Section( Field(MATRIX_TOKEN, "Access-Token des Bot-Users", kind="password", secret=True), Field(MATRIX_ROOM_ID, "Standard-Raum-ID (unverschlüsselt)", placeholder="!abcdef:example.de"), Field(MATRIX_ALLOWLIST, "Erlaubte Matrix-IDs (kommagetrennt)", placeholder="@admin:example.de"), + content_mode_field(MATRIX_CONTENT_MODE, + "Mit \"kurz\" enthalten Nachrichten keinen Plugin-Output und keine KI-Texte; " + "die KI-Analyse wird dann nicht nachgereicht."), ), validate=validate, test=test_connection, diff --git a/app/web/problem_status.py b/app/web/problem_status.py index 4f979be..fafc247 100644 --- a/app/web/problem_status.py +++ b/app/web/problem_status.py @@ -3,7 +3,8 @@ import time from fastapi import Request from ..alerts import report_status -from ..poller.options import read_default_delay +from ..models import HOST_SERVICE_NAME +from ..poller.options import read_default_delay, read_host_delay def problems_with_status(request: Request) -> list[dict]: @@ -11,6 +12,9 @@ def problems_with_status(request: Request) -> list[dict]: now = time.time() rules = runtime.routes.enabled_routes() mutes = runtime.mutes.active(now) - default_minutes = read_default_delay(request.app.state.store) - return [{**row, "report": report_status(row, rules, mutes, default_minutes, now)} - for row in runtime.alerts.open_problems()] + store = request.app.state.store + default_minutes, host_minutes = read_default_delay(store), read_host_delay(store) + rows = runtime.alerts.open_problems() + down_hosts = {row["host"] for row in rows if row["service"] == HOST_SERVICE_NAME} + return [{**row, "report": report_status(row, rules, mutes, default_minutes, now, host_minutes, down_hosts)} + for row in rows] diff --git a/app/web/sections.py b/app/web/sections.py index 578d36e..39e3a3b 100644 --- a/app/web/sections.py +++ b/app/web/sections.py @@ -18,6 +18,9 @@ class Field: secret: bool = False placeholder: str = "" default: str = "" + choices: tuple[tuple[str, str], ...] = () + hint: str = "" + transform: Callable[[str], str] | None = None @property def name(self) -> str: @@ -63,6 +66,8 @@ def collect(form, section: Section) -> dict[str, str | None]: submitted[field.key] = FLAG_ON if form.get(field.name) else FLAG_OFF continue raw = str(form.get(field.name, "")).strip() + if field.transform is not None: + raw = field.transform(raw) submitted[field.key] = (raw or None) if field.secret else raw return submitted diff --git a/app/web/telegram_section.py b/app/web/telegram_section.py index 062e7f7..65ce035 100644 --- a/app/web/telegram_section.py +++ b/app/web/telegram_section.py @@ -6,8 +6,10 @@ from fastapi.responses import JSONResponse from .. import audit from ..chat import TELEGRAM_CHAT_ID_RE, TelegramAdapter, mask_secret from ..constants import (DEFAULT_TELEGRAM_API_BASE, FLAG_OFF, FLAG_ON, TELEGRAM_ALLOWLIST, - TELEGRAM_API_BASE, TELEGRAM_CHAT_ID, TELEGRAM_ENABLED, TELEGRAM_TOKEN) + TELEGRAM_API_BASE, TELEGRAM_CHAT_ID, TELEGRAM_CONTENT_MODE, TELEGRAM_ENABLED, + TELEGRAM_TOKEN) from ..users import User +from .content_mode import content_mode_field, validate_content_mode from .deps import require_admin from .sections import URL_RE, Action, Field, Section, check, collect, effective @@ -40,7 +42,7 @@ def validate(values: dict) -> list[str]: api_base = values.get(TELEGRAM_API_BASE, "") if api_base and not URL_RE.match(api_base): errors.append("API-Basis-URL muss mit http:// oder https:// beginnen.") - return errors + return errors + validate_content_mode(values, TELEGRAM_CONTENT_MODE) def _adapter(values: dict) -> TelegramAdapter: @@ -68,6 +70,9 @@ SECTION = Section( Field(TELEGRAM_CHAT_ID, "Standard-Chat-ID", placeholder="123456789 oder @kanalname"), Field(TELEGRAM_ALLOWLIST, "Erlaubte Benutzer-IDs (kommagetrennt)", placeholder="111111,222222"), Field(TELEGRAM_API_BASE, "API-Basis-URL (optional)", placeholder=DEFAULT_TELEGRAM_API_BASE), + content_mode_field(TELEGRAM_CONTENT_MODE, + "Die Telegram-Server liegen außerhalb. Mit \"kurz\" verlassen weder Plugin-Output " + "noch KI-Texte das Haus; die KI-Analyse wird dann nicht nachgereicht."), ), validate=validate, test=test_connection, diff --git a/app/web/templates/_route_form.html b/app/web/templates/_route_form.html index 059a4ce..8051023 100644 --- a/app/web/templates/_route_form.html +++ b/app/web/templates/_route_form.html @@ -34,7 +34,7 @@ Verhalten - + diff --git a/app/web/templates/alerts.html b/app/web/templates/alerts.html index 5c1f541..33da890 100644 --- a/app/web/templates/alerts.html +++ b/app/web/templates/alerts.html @@ -8,12 +8,12 @@ ZeitHostServiceArtStatusOutputKI {% for a in entries %} - + {{ a.ts | fmt_ts }} {{ a.host }} {{ a.service }} {{ a.kind }} - {{ a.state | state_name }} + {{ a.state | display_state(a.service) }} {{ a.output }}{% if a.suppressed %} unterdrückt ({{ suppress_labels.get(a.suppress_reason, a.suppress_reason) }}){% endif %} {{ ai_labels.get(a.ai_status, a.ai_status) }} diff --git a/app/web/templates/dashboard.html b/app/web/templates/dashboard.html index 9eb18a3..48f072c 100644 --- a/app/web/templates/dashboard.html +++ b/app/web/templates/dashboard.html @@ -34,7 +34,7 @@ {{ p.host }} {{ p.service }} - {{ p.state | state_name }} + {{ p.state | display_state(p.service) }} {{ p.last_state_change | fmt_ts }} {{ p.report.label }} {{ p.output }} diff --git a/app/web/templates/routing.html b/app/web/templates/routing.html index e381150..b7606ae 100644 --- a/app/web/templates/routing.html +++ b/app/web/templates/routing.html @@ -20,7 +20,7 @@ {% if r.output_contains %}
Ausgabe enthält „{{ r.output_contains }}“
{% endif %}
Verzögerung: {{ (r.delay_minutes ~ ' min') if r.delay_minutes is not none else 'Standard' }}
KI: {{ 'an' if r.ai_enabled else 'aus' }}{% if r.ai_hint %} (mit Hinweis){% endif %}
- {% if r.reminder_hours %}
Erinnerung: alle {{ r.reminder_hours }} h
{% endif %} + {% if r.reminder_hours is not none %}
Erinnerung: {{ ('alle ' ~ r.reminder_hours ~ ' h') if r.reminder_hours > 0 else 'aus' }}
{% endif %} {% if not r.loud %}
leise
{% endif %} {{ 'aktiv' if r.enabled else 'aus' }} diff --git a/app/web/templates/settings.html b/app/web/templates/settings.html index 0a58ae8..f665811 100644 --- a/app/web/templates/settings.html +++ b/app/web/templates/settings.html @@ -12,6 +12,11 @@ {{ field.label }} + {% elif field.kind == "select" %} + + {% elif field.secret %} {% endif %} + {% if field.hint %}

{{ field.hint }}

{% endif %} {% endfor %}
diff --git a/deploy/README.md b/deploy/README.md index 436ad3e..42bc47a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -103,23 +103,17 @@ Der Bot schreibt Zeilen nach stderr, journald stempelt die Zeit (`journalctl -u ## Erster Testlauf -Vor dem ersten echten Betrieb die Tests im Test-LXC laufen lassen, in einer eigenen Dev-venv und nicht unter `/opt/checkbot` (dort liegen weder Tests noch Dev-Abhängigkeiten): +Vor dem ersten echten Betrieb die Tests im Test-LXC laufen lassen. `deploy/first-run.sh` braucht kein root, legt eine eigene Dev-venv an (nicht unter `/opt/checkbot`, dort liegen weder Tests noch Dev-Abhängigkeiten) und erledigt alles in einem Lauf: ```bash -apt-get install -y python3-venv python3-pip -cd /root/checkbot -python3 -m venv /root/checkbot-devenv -/root/checkbot-devenv/bin/pip install -r requirements-dev.txt -/root/checkbot-devenv/bin/pytest -x -q +apt-get install -y python3-venv python3-pip # einmalig, als root +cd ~/checkbot +deploy/first-run.sh # optional: --venv --out ``` -Für eine vollständige Fehlerausgabe ohne `-x` und zum Sichern: +Ablauf: venv (Standard `$HOME/checkbot-test-venv`) anlegen, `pip install -r requirements-dev.txt`, `import app.main`, alle Jinja-Templates mit der App-Umgebung kompilieren, danach `pytest -q` **ohne** `-x`, damit alle Fehler in einer Runde zusammenkommen. Zum Schluss packt das Skript `first-run-.tar.gz` (in `--out`, Standard das aktuelle Verzeichnis) mit pytest-Log, Import- und Template-Log, Python-Versionen, `pip freeze`, `uname -a` und der Git-Revision (falls ein Checkout vorliegt). Das Archiv enthält keine Umgebungsvariablen, Datenbanken, Schlüsseldateien und nichts aus `/etc/checkbot`. -```bash -/root/checkbot-devenv/bin/pytest -q 2>&1 | tee pytest-run.log -``` - -Die Datei `pytest-run.log` geht unverändert an den Entwickler zurück (alle Fehler gesammelt, damit sie in einer Runde korrigiert werden). Integrationstests (`-m integration`) laufen nie versehentlich mit, sie sind standardmäßig ausgeschlossen. +Das Archiv geht unverändert an den Entwickler zurück. Integrationstests (`-m integration`) laufen nie versehentlich mit, sie sind standardmäßig ausgeschlossen. ## 4. Update diff --git a/deploy/first-run.sh b/deploy/first-run.sh new file mode 100755 index 0000000..b168d62 --- /dev/null +++ b/deploy/first-run.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Erster Testlauf: Dev-venv anlegen, Import und Templates prüfen, pytest vollständig laufen lassen +# und die Ergebnisse (ohne Secrets) in ein Archiv packen. Braucht kein root. +set -euo pipefail + +VENV="${HOME}/checkbot-test-venv" +OUT="$(pwd)" +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +usage() { + cat <] [--out ] + + --venv Dev-venv (Standard: \$HOME/checkbot-test-venv) + --out Zielverzeichnis des Archivs (Standard: aktuelles Verzeichnis) +EOF +} + +die() { + echo "FEHLER: $*" >&2 + exit 1 +} + +while [ $# -gt 0 ]; do + case "$1" in + --venv) [ $# -ge 2 ] || die "--venv braucht ein Verzeichnis"; VENV="$2"; shift 2 ;; + --out) [ $# -ge 2 ] || die "--out braucht ein Verzeichnis"; OUT="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; die "Unbekannte Option: $1" ;; + esac +done + +[ -f "$REPO_DIR/requirements-dev.txt" ] || die "requirements-dev.txt nicht gefunden in $REPO_DIR" +command -v python3 >/dev/null 2>&1 || die "python3 fehlt (apt-get install python3 python3-venv python3-pip)" +mkdir -p "$OUT" || die "Zielverzeichnis $OUT nicht anlegbar" +OUT="$(cd "$OUT" && pwd)" + +WORK="$(mktemp -d)" +PIP_LOG="$(mktemp)" +trap 'rm -rf "$WORK" "$PIP_LOG"' EXIT + +echo "== 1/6 Dev-venv: $VENV" +if [ ! -x "$VENV/bin/python" ]; then + python3 -m venv "$VENV" || die "venv konnte nicht angelegt werden (apt-get install python3-venv)" +fi + +echo "== 2/6 Abhängigkeiten installieren (requirements-dev.txt)" +if ! "$VENV/bin/python" -m pip install --quiet -r "$REPO_DIR/requirements-dev.txt" >"$PIP_LOG" 2>&1; then + tail -n 30 "$PIP_LOG" >&2 + die "pip install fehlgeschlagen" +fi + +cd "$REPO_DIR" + +echo "== 3/6 Import von app.main" +IMPORT_STATUS=ok +"$VENV/bin/python" -c "import app.main" >"$WORK/import.log" 2>&1 || IMPORT_STATUS=FEHLER +echo " Ergebnis: $IMPORT_STATUS" + +echo "== 4/6 Jinja-Templates kompilieren" +TEMPLATE_STATUS=ok +"$VENV/bin/python" - >"$WORK/templates.log" 2>&1 <<'PY' || TEMPLATE_STATUS=FEHLER +from app.web.deps import templates + +env = templates.env +names = env.list_templates() +failed = 0 +for name in names: + try: + env.get_template(name) + except Exception as exc: + failed += 1 + print(f"FEHLER {name}: {type(exc).__name__}: {exc}") +print(f"{len(names)} Templates, {failed} Fehler") +raise SystemExit(1 if failed else 0) +PY +echo " Ergebnis: $TEMPLATE_STATUS" + +echo "== 5/6 pytest -q (ohne -x, alle Fehler werden gesammelt)" +PYTEST_STATUS=ok +"$VENV/bin/python" -m pytest -q >"$WORK/pytest.log" 2>&1 || PYTEST_STATUS=FEHLER +tail -n 3 "$WORK/pytest.log" | sed 's/^/ /' + +echo "== 6/6 Archiv packen" +{ + echo "venv-python: $("$VENV/bin/python" --version 2>&1)" + echo "system-python: $(python3 --version 2>&1)" +} >"$WORK/python-version.txt" +"$VENV/bin/python" -m pip freeze >"$WORK/pip-freeze.txt" 2>&1 || true +uname -a >"$WORK/uname.txt" +if git -C "$REPO_DIR" rev-parse --git-dir >/dev/null 2>&1; then + { + echo "revision: $(git -C "$REPO_DIR" rev-parse HEAD 2>/dev/null || echo unbekannt)" + echo "branch: $(git -C "$REPO_DIR" rev-parse --abbrev-ref HEAD 2>/dev/null || echo unbekannt)" + if [ -n "$(git -C "$REPO_DIR" status --porcelain 2>/dev/null)" ]; then + echo "arbeitsverzeichnis: geaendert (nicht committete Aenderungen)" + else + echo "arbeitsverzeichnis: sauber" + fi + } >"$WORK/git-revision.txt" +fi +{ + echo "import: $IMPORT_STATUS" + echo "templates: $TEMPLATE_STATUS" + echo "pytest: $PYTEST_STATUS" +} >"$WORK/summary.txt" + +ARCHIVE="$OUT/first-run-$(date +%Y%m%d-%H%M%S).tar.gz" +tar -czf "$ARCHIVE" -C "$WORK" . + +echo +echo "Import: $IMPORT_STATUS | Templates: $TEMPLATE_STATUS | pytest: $PYTEST_STATUS" +echo "Archiv: $ARCHIVE" +echo "Das Archiv enthält nur Logs und Versionsangaben (keine Umgebungsvariablen, Datenbanken oder Schlüssel)." diff --git a/tests/test_checkmk_hosts.py b/tests/test_checkmk_hosts.py new file mode 100644 index 0000000..469607b --- /dev/null +++ b/tests/test_checkmk_hosts.py @@ -0,0 +1,110 @@ +import json + +import httpx +import pytest + +from app.checkmk import (COLUMNS, FAILURE_UNEXPECTED, FAILURE_UNREACHABLE, HOST_COLUMNS, CheckmkClient, + CheckmkError, CheckmkUnreachable) +from app.models import HOST_SERVICE_NAME + +BASE = "https://cmk.local/prod/check_mk/api/1.0" + + +def make_client(handler): + return CheckmkClient("https://cmk.local", "prod", "bot", "s3cret", transport=httpx.MockTransport(handler)) + + +def host_item(name="srv1", state=1, **extra): + data = {"name": name, "state": state, "plugin_output": "DOWN - no route", "last_state_change": 42, + "scheduled_downtime_depth": 0, "acknowledged": 0} + data.update(extra) + return {"extensions": data} + + +async def test_problem_hosts_posts_query_and_columns(): + seen = {} + + def handler(request): + seen["url"] = str(request.url) + seen["body"] = json.loads(request.content) + return httpx.Response(200, json={"value": [host_item()]}) + + hosts = await make_client(handler).problem_hosts() + assert seen["url"] == f"{BASE}/domain-types/host/collections/all" + assert seen["body"]["columns"] == HOST_COLUMNS + assert seen["body"]["query"] == {"op": "!=", "left": "state", "right": "0"} + assert [(h.host, h.state, h.output, h.last_state_change) for h in hosts] == [("srv1", 2, "DOWN - no route", 42)] + + +async def test_host_states_map_to_internal_numbers(): + body = {"value": [host_item("a", 1), host_item("b", 2)]} + hosts = await make_client(lambda r: httpx.Response(200, json=body)).problem_hosts() + assert [h.state for h in hosts] == [2, 3] + + +async def test_host_downtime_and_ack_flags(): + body = {"value": [host_item(scheduled_downtime_depth=1, acknowledged=1)]} + (host,) = await make_client(lambda r: httpx.Response(200, json=body)).problem_hosts() + assert host.in_downtime and host.acknowledged + + +async def test_host_as_service_uses_host_service_name_and_is_hard(): + (host,) = await make_client(lambda r: httpx.Response(200, json={"value": [host_item(acknowledged=1)]}) + ).problem_hosts() + service = host.as_service() + assert service.service == HOST_SERVICE_NAME and service.hard and service.acknowledged and not service.flapping + + +async def test_service_columns_include_acknowledged_and_flag_is_parsed(): + assert "acknowledged" in COLUMNS + item = {"extensions": {"host_name": "h", "description": "s", "state": 2, "state_type": 1, "plugin_output": "x", + "last_state_change": 1, "scheduled_downtime_depth": 0, "is_flapping": 0, + "acknowledged": 1}} + (service,) = await make_client(lambda r: httpx.Response(200, json={"value": [item]})).problem_services() + assert service.acknowledged is True + + +async def test_missing_acknowledged_column_means_not_acknowledged(): + item = {"extensions": {"host_name": "h", "description": "s", "state": 2}} + (service,) = await make_client(lambda r: httpx.Response(200, json={"value": [item]})).problem_services() + assert service.acknowledged is False + + +def failing(error): + def handler(request): + raise error + return handler + + +async def test_transport_error_is_unreachable(): + with pytest.raises(CheckmkUnreachable) as caught: + await make_client(failing(httpx.ConnectError("boom"))).problem_services() + assert caught.value.kind == FAILURE_UNREACHABLE and caught.value.brief == "ConnectError" + + +async def test_timeout_is_unreachable(): + with pytest.raises(CheckmkError) as caught: + await make_client(failing(httpx.ReadTimeout("slow"))).problem_hosts() + assert caught.value.kind == FAILURE_UNREACHABLE + + +async def test_http_5xx_is_unreachable_and_4xx_is_unexpected(): + with pytest.raises(CheckmkError) as caught: + await make_client(lambda r: httpx.Response(503, text="busy")).problem_services() + assert caught.value.kind == FAILURE_UNREACHABLE and caught.value.brief == "HTTP 503" + with pytest.raises(CheckmkError) as caught: + await make_client(lambda r: httpx.Response(401, text="geheim: s3cret")).problem_services() + assert caught.value.kind == FAILURE_UNEXPECTED and caught.value.brief == "HTTP 401" + assert "s3cret" not in caught.value.brief + + +async def test_non_json_and_wrong_schema_are_unexpected(): + with pytest.raises(CheckmkError) as caught: + await make_client(lambda r: httpx.Response(200, text="")).problem_services() + assert caught.value.kind == FAILURE_UNEXPECTED + with pytest.raises(CheckmkError) as caught: + await make_client(lambda r: httpx.Response(200, json={"value": [{"extensions": {"foo": 1}}]}) + ).problem_services() + assert caught.value.kind == FAILURE_UNEXPECTED and caught.value.brief == "Antwortformat unerwartet" + with pytest.raises(CheckmkError): + await make_client(lambda r: httpx.Response(200, json=[1, 2])).problem_hosts() diff --git a/tests/test_db_migrations.py b/tests/test_db_migrations.py new file mode 100644 index 0000000..30cb05e --- /dev/null +++ b/tests/test_db_migrations.py @@ -0,0 +1,101 @@ +import pytest + +from app.db import MIGRATIONS, Database, MigrationError +from app.migrate import split_statements + +CREATE_A = "CREATE TABLE a (id INTEGER PRIMARY KEY, name TEXT);" +GOOD_SECOND = "CREATE TABLE b (id INTEGER); INSERT INTO a(name) VALUES ('x');" +BROKEN_SECOND = "CREATE TABLE c (id INTEGER); INSERT INTO missing_table VALUES (1);" + + +def version(db): + return db.query("PRAGMA user_version")[0][0] + + +def tables(db): + return {row["name"] for row in db.query("SELECT name FROM sqlite_master WHERE type='table'")} + + +def test_split_statements_keeps_quoted_semicolons_and_drops_empty_tail(): + script = "CREATE TABLE t (v TEXT);\nINSERT INTO t VALUES ('a;b');\n \n" + assert split_statements(script) == ["CREATE TABLE t (v TEXT);", "INSERT INTO t VALUES ('a;b');"] + + +def test_successful_run_sets_user_version(): + db = Database(":memory:", migrations=[CREATE_A, GOOD_SECOND]) + assert version(db) == 2 + assert {"a", "b"} <= tables(db) + assert db.query("SELECT COUNT(*) AS n FROM a")[0]["n"] == 1 + + +def test_default_migrations_apply_completely(): + db = Database(":memory:") + assert version(db) == len(MIGRATIONS) + assert {"service_states", "routes", "app_state"} <= tables(db) + columns = {row["name"] for row in db.query("PRAGMA table_info(service_states)")} + assert {"ack_state", "host_held"} <= columns + + +def test_broken_second_statement_rolls_back_first_and_keeps_version(tmp_path): + path = tmp_path / "m.db" + Database(path, migrations=[CREATE_A]).close() + with pytest.raises(MigrationError, match="Migration 1"): + Database(path, migrations=[CREATE_A, BROKEN_SECOND]) + db = Database(path, migrations=[CREATE_A]) + assert version(db) == 1 + assert "c" not in tables(db) + + +def test_existing_tables_and_data_survive_failed_follow_up_migration(tmp_path): + path = tmp_path / "m.db" + db = Database(path, migrations=[CREATE_A]) + db.insert("INSERT INTO a(name) VALUES ('keep')") + db.close() + with pytest.raises(MigrationError): + Database(path, migrations=[CREATE_A, BROKEN_SECOND]) + again = Database(path, migrations=[CREATE_A]) + assert [r["name"] for r in again.query("SELECT name FROM a")] == ["keep"] + + +def test_restart_does_not_apply_migrations_twice(tmp_path): + path = tmp_path / "m.db" + migrations = [CREATE_A, "INSERT INTO a(name) VALUES ('once');"] + Database(path, migrations=migrations).close() + db = Database(path, migrations=migrations) + assert version(db) == 2 + assert db.query("SELECT COUNT(*) AS n FROM a")[0]["n"] == 1 + + +def test_failed_migration_can_be_retried_after_fix(tmp_path): + path = tmp_path / "m.db" + Database(path, migrations=[CREATE_A]).close() + with pytest.raises(MigrationError): + Database(path, migrations=[CREATE_A, BROKEN_SECOND]) + db = Database(path, migrations=[CREATE_A, GOOD_SECOND]) + assert version(db) == 2 + + +def test_connection_keeps_normal_commit_behaviour_after_migration(): + db = Database(":memory:", migrations=[CREATE_A]) + db.insert("INSERT INTO a(name) VALUES ('p')") + with pytest.raises(Exception): + db.execute_many([("INSERT INTO a(name) VALUES ('q')", ()), ("INSERT INTO nope VALUES (1)", ())]) + assert [r["name"] for r in db.query("SELECT name FROM a")] == ["p"] + + +def test_phase_6c_migrations_keep_rows_set_marker_and_null_zero_reminders(tmp_path): + path = tmp_path / "m.db" + old = Database(path, migrations=MIGRATIONS[:7]) + old.execute("INSERT INTO service_states(host, service, state, last_state_change) VALUES ('h','s',2,1)") + old.execute("INSERT INTO routes(name, position, reminder_hours) VALUES ('a', 1, 0)") + old.execute("INSERT INTO routes(name, position, reminder_hours) VALUES ('b', 2, 4)") + old.close() + db = Database(path) + row = db.query("SELECT ack_state, host_held FROM service_states")[0] + assert row["ack_state"] is None and row["host_held"] == 0 + assert db.query("SELECT value FROM app_state WHERE key='baseline.done'")[0]["value"] == "1" + assert [r["reminder_hours"] for r in db.query("SELECT reminder_hours FROM routes ORDER BY position")] == [None, 4] + + +def test_fresh_database_has_no_baseline_marker(): + assert Database(":memory:").query("SELECT 1 FROM app_state WHERE key='baseline.done'") == [] diff --git a/tests/test_formatter_host_short.py b/tests/test_formatter_host_short.py new file mode 100644 index 0000000..d549db7 --- /dev/null +++ b/tests/test_formatter_host_short.py @@ -0,0 +1,95 @@ +from datetime import datetime + +from app.alerts import Analysis, build_user_prompt, format_message +from app.models import (HOST_SERVICE_NAME, KIND_FLAPPING, KIND_PROBLEM, KIND_RECOVERY, KIND_REMINDER, Event, + display_state, severity) + +NOW = int(datetime(2026, 11, 6, 20, 0).timestamp()) +SINCE = int(datetime(2026, 11, 6, 18, 5).timestamp()) +HOUR = 3600 + + +def host_event(kind=KIND_PROBLEM, state=2, **kwargs): + return Event(ts=NOW, host="srv1", service=HOST_SERVICE_NAME, kind=kind, state=state, output="no route", **kwargs) + + +def service_event(kind=KIND_PROBLEM, state=2, **kwargs): + return Event(ts=NOW, host="srv1", service="zpool", kind=kind, state=state, output="kaputt", **kwargs) + + +def test_display_state_names_hosts_and_services(): + assert [display_state(HOST_SERVICE_NAME, s) for s in (0, 2, 3)] == ["UP", "DOWN", "UNREACH"] + assert [display_state("cpu", s) for s in (0, 1, 2, 3)] == ["OK", "WARN", "CRIT", "UNKNOWN"] + + +def test_severity_orders_crit_before_unknown_before_warn(): + assert severity(2) > severity(3) > severity(1) > severity(0) + + +def test_host_down_line(): + text = format_message([host_event(since_ts=SINCE)]) + assert "srv1" in text and "Host DOWN, seit 18:05: no route" in text + assert "Host" in text and "zpool" not in text + + +def test_host_unreach_line_and_transition(): + assert "Host UNREACH: no route" in format_message([host_event(state=3)]) + text = format_message([host_event(state=3, previous_state=2, previous_duration=HOUR, since_ts=SINCE)]) + assert "Host DOWN → UNREACH, seit 18:05 (vorher DOWN seit 1 h): no route" in text + + +def test_host_recovery_text(): + event = host_event(kind=KIND_RECOVERY, state=0, previous_state=2, problem_since_ts=NOW - (3 * HOUR + 12 * 60)) + assert "Host wieder UP nach 3 h 12 min (zuletzt DOWN)" in format_message([event]) + + +def test_host_recovery_without_delta(): + assert "Host wieder UP" in format_message([host_event(kind=KIND_RECOVERY, state=0)]) + + +def test_host_reminder_line(): + text = format_message([host_event(kind=KIND_REMINDER, since_ts=NOW - 4 * HOUR)]) + assert "Host DOWN: weiter DOWN seit 4 h: no route" in text + + +def test_host_output_is_escaped(): + event = Event(ts=NOW, host="", service=HOST_SERVICE_NAME, kind=KIND_PROBLEM, state=2, + output="") + text = format_message([event]) + assert "