From 453c077386ad3267d6ff2c4fc14009923709a729 Mon Sep 17 00:00:00 2001 From: patrick Date: Fri, 6 Nov 2026 12:55:57 +0100 Subject: [PATCH] feat: Phase 6d API-Selbsttest gegen Checkmk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lesender Selbsttest (Version, Services, Hosts, Downtimes) beim Start und täglich 04:00, Seite /selftest, Teilcheck checkmk_api in /health, Chat-Meldung bei Wechsel. Toleranter Antwort-Parser (value/members, state_type int/String), Fallbacks für Host-Abfrage und Downtime-Liste, Fehlerhinweise je HTTP-Status. Co-Authored-By: Claude Sonnet 5.5 --- CLAUDE.md | 19 ++- README.md | 7 ++ app/alerts/system.py | 7 +- app/alerts/system_text.py | 1 + app/checkmk.py | 126 ++++++++++++++++++-- app/constants.py | 3 + app/health.py | 21 +++- app/main.py | 6 +- app/migrations.py | 8 ++ app/runtime.py | 14 ++- app/selftest/__init__.py | 15 +++ app/selftest/checks.py | 92 ++++++++++++++ app/selftest/constants.py | 25 ++++ app/selftest/model.py | 31 +++++ app/selftest/notify.py | 20 ++++ app/selftest/repository.py | 25 ++++ app/selftest/runner.py | 88 ++++++++++++++ app/selftest/service.py | 43 +++++++ app/web/pages.py | 6 +- app/web/selftest_pages.py | 32 +++++ app/web/selftest_section.py | 21 ++++ app/web/templates/base.html | 2 + app/web/templates/selftest.html | 37 ++++++ deploy/README.md | 8 +- deploy/checkmk/checkbot-local-check | 8 +- tests/test_checkmk_fallbacks.py | 120 +++++++++++++++++++ tests/test_checkmk_reads.py | 25 ++++ tests/test_health_selftest.py | 71 +++++++++++ tests/test_selftest.py | 179 ++++++++++++++++++++++++++++ tests/test_selftest_runner.py | 142 ++++++++++++++++++++++ tests/test_web_selftest.py | 78 ++++++++++++ 31 files changed, 1249 insertions(+), 31 deletions(-) create mode 100644 app/selftest/__init__.py create mode 100644 app/selftest/checks.py create mode 100644 app/selftest/constants.py create mode 100644 app/selftest/model.py create mode 100644 app/selftest/notify.py create mode 100644 app/selftest/repository.py create mode 100644 app/selftest/runner.py create mode 100644 app/selftest/service.py create mode 100644 app/web/selftest_pages.py create mode 100644 app/web/selftest_section.py create mode 100644 app/web/templates/selftest.html create mode 100644 tests/test_checkmk_fallbacks.py create mode 100644 tests/test_health_selftest.py create mode 100644 tests/test_selftest.py create mode 100644 tests/test_selftest_runner.py create mode 100644 tests/test_web_selftest.py diff --git a/CLAUDE.md b/CLAUDE.md index d734483..1461e68 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ Vollständiger Plan: `/home/sysops/.claude/plans/ja-alle-bernehmen-eager-lark.md ## Arbeitsregeln - Auf diesem Rechner wird nichts installiert (kein venv, kein pip, keine Pakete). Code nur schreiben, Ausführung und Tests passieren im Ziel-LXC. -- Stand: Phase 1 bis 6, 6b und 6c als Code, Phase 7 als Dateien geschrieben, nichts ausgeführt oder getestet (siehe README-Statustabelle). +- Stand: Phase 1 bis 6, 6b, 6c und 6d als Code, Phase 7 als Dateien geschrieben, nichts ausgeführt oder getestet (siehe README-Statustabelle). - Code-Kommentare nur bei echtem WHY. ## Festgelegte Entscheidungen @@ -91,6 +91,17 @@ tests/ - Annahmen: Host-Abruf `POST /domain-types/host/collections/all` mit Query `state != 0` und Spalten `name`, `state`, `plugin_output`, `last_state_change`, `scheduled_downtime_depth`, `acknowledged`; Service-Spalte `acknowledged`; Host-State 0/1/2 = UP/DOWN/UNREACH (intern 0/2/3); Hosts gelten immer als hart. Alles ungeprüft gegen 2.5 (Phase 6d prüft). - Abweichung: Datenschutzmodus gilt je Backend, nicht je einzelnem Ziel (Ziele stehen in Regeln als freie Zeilen). Ein vom Sendelimit verworfenes Problem zählt im Speicher des Gates als gemeldet (die Recovery kommt). +## Phase 6d API-Selbsttest (Code geschrieben, ungetestet) +- Migration Index 9 (zehnter Eintrag in `app/migrations.py`): Tabelle `selftest_state` (Einzelzeile id=1: ts, version, checks als JSON-Liste `{name, status, text}`), analog zu `retention_state`. +- Neues Modul `app/selftest/`: `constants.py` (Status `ok`/`abweichend`/`fehler`, Teilcheck-Namen `version`/`services`/`hosts`/`downtimes`, Zeitplan-Konstanten), `model.py` (`CheckResult`, `SelftestRun` mit `overall_status`/`ok`), `repository.py` (`SelftestRepository`, liest/schreibt die Einzelzeile), `checks.py` (die vier Teilchecks, rein lesend, unterscheiden `nicht erreichbar` von `Antwort unerwartet` über `CheckmkUnreachable`/`CheckmkError`, prüfen Pflicht- und optionale Spalten im rohen JSON vor dem Parsen), `service.py` (`SelftestService.run()` ruft die vier Teilchecks auf, vergleicht die Version mit dem letzten Lauf, speichert), `notify.py` (`message_for`: Text nur bei Statuswechsel, Rückkehr zu ok oder Versionswechsel, sonst `None`), `runner.py` (`SelftestRunner`: Zeitplan wie `RetentionRunner`, aber Start-Verzögerung 20 s statt 60 s und täglich ab 04:00 statt 03:00; manueller Lauf über `run_manual` mit Audit `selftest.run`, Rate-Limit 1 pro 30 s über `SlidingWindowLimiter`). +- `app/checkmk.py`: `_items`/`extract_extensions` akzeptieren `value` oder `members`; `_is_hard` über die benannte Konstante `HARD_STATE_TYPES` (`"1"`/`"hard"`, alles andere inkl. `None` gilt als soft, stürzt nicht ab); `CheckmkError` bekommt zusätzlich `.hint` (lesbarer Hinweis je HTTP-Status 401/403/404/422/5xx über `status_hint`), `.brief` bleibt wortgleich wie bisher, damit bestehende Tests und Chat-Texte unverändert bleiben. Neue, nur für den Selbsttest gedachte Zusatzpfade, die das reguläre Verhalten nicht verändern: `problem_hosts_with_fallback`/`selftest_hosts` (POST scheitert → GET mit `columns=` und clientseitigem Filter `state != 0`, der funktionierende Weg wird in `_host_query_mode` gemerkt und von `problem_hosts()` beim nächsten Aufruf zuerst versucht), `selftest_services` (rohe Zeilen plus best-effort Parse, Parsefehler werden abgefangen statt den Selbsttest abzubrechen), `downtimes_with_fallback` (scheitert `downtime_type` → GET mit `query` als JSON-String, `is_service` 0/1), `downtimes_from_states` (reine Funktion, leitet Downtimes aus `scheduled_downtime_depth` der schon gelesenen Service-/Host-Listen ab, letzter Ausweg). +- `app/health.py`: neuer Teilcheck `checkmk_api`, liest nur den letzten gespeicherten Selbsttest-Lauf (kein eigener API-Aufruf). Checkmk nicht konfiguriert oder kein `selftest_repo` übergeben → `NotConfigured`; noch kein Lauf → `NotConfigured("noch nicht gelaufen")` (gleiche Behandlung wie "nicht konfiguriert", dafür liest `run_checks` jetzt den Text aus der Exception statt ihn fest zu verdrahten); Status `abweichend` oder `fehler` → Fehler (503 in `/health`). `build_checks(store, poller_status=None, selftest_repo=None)` ist rückwärtskompatibel (fehlender `selftest_repo` ergibt stets "nicht konfiguriert"). +- `app/runtime.py`: `Runtime.selftest_repo`/`.selftest`/`.selftest_runner`, Task im Lifespan wie `retention_task` (Cancel in `finally`). `app/alerts/system.py` bekommt `SystemNotifier.selftest(ok, text)` (eigener Service-Name `Selbsttest` in `system_text.py`, Art `system`, wie die Checkmk-Ausfallmeldung). +- Settings `selftest.enabled` (Default an, schaltet nur den automatischen Zeitplan ab, ein manueller Lauf funktioniert immer) und `selftest.notify` (Default an) in neuer Section "Selbsttest" (`app/web/selftest_section.py`, `/settings/selftest`). +- UI `/selftest` (`app/web/selftest_pages.py` + `selftest.html`): Tabelle der vier Teilchecks mit Ampel-Badge, Checkmk-Version, Zeitpunkt des letzten Laufs; Fallback-Hinweise stehen im Text des jeweiligen Teilchecks. Lesbar für alle angemeldeten Rollen, Button "Selbsttest jetzt ausführen" nur für Admins (CSRF über die globale FastAPI-Dependency, Rate-Limit 1 pro 30 s). Menüeinträge `/selftest` (alle) und `/settings/selftest` (Admins). +- `deploy/checkmk/checkbot-local-check`: neuer Service `Checkbot_checkmk_api`, Detailtext `noch nicht gelaufen` wird wie `nicht konfiguriert` als WARN gewertet (`WARN_DETAILS`). +- Annahmen/Abweichungen: "sofort bei Versionswechsel" löst keinen zusätzlichen Sofort-Lauf aus, sondern markiert nur den laufenden Lauf als Versionswechsel für die Chat-Meldung (im Auftrag so vorgesehen: "sofort erneut laufen lassen bzw. den aktuellen Lauf kennzeichnen"). Fehlt der Parser-Vergleich an Spalten, weil aktuell keine Problem-Services/-Hosts vorliegen (leere Ergebnisliste), kann der jeweilige Teilcheck die Spalten nicht prüfen und meldet `ok`, auch wenn eine Spalte in Wahrheit fehlen würde – eine Grenze des Ansatzes, der nur die echten Problem-Listen liest statt eines eigenen Schema-Endpunkts. `.hint` an `CheckmkError` ist zusätzlich zu `.brief`, nicht statt dessen, damit bestehende Tests mit exaktem `brief`-Text (z. B. `"HTTP 401"`) unverändert bleiben. + ## Weiterer Weg (entschieden) Reihenfolge: Testbasis, erster Lauf im Test-LXC (`pytest`), Korrekturrunde, Phase 6, Phase 6b, Phase 6c, Phase 8, Abnahme, Parallelbetrieb, Tag `v0.1.0`. Verfeinerungen passieren am Live-System, nicht vorab. - Testbasis: `pytest` manuell im Test-LXC (Dev-venv mit `requirements-dev.txt`), dazu `.gitea/workflows/test.yml` für einen späteren Gitea-Actions-Runner. Logging als Zeilenlog nach journald, Level über `CHECKBOT_LOG_LEVEL` (Default INFO), nie Secrets in Logs (Test für die Adapter). Zeitzone über `CHECKBOT_TZ` im Setup-Skript, Default `Europe/Berlin`. @@ -113,9 +124,9 @@ Reihenfolge: Testbasis, erster Lauf im Test-LXC (`pytest`), Korrekturrunde, Phas - Delta in der Alarmnachricht (Phase 6b): `Event` bekommt Vorzustand und Dauer. Alarmzeile zeigt Übergang und Dauer (`WARN → CRIT, seit 18:05 (vorher WARN seit 2 h)`), Recovery die Gesamtdauer (`wieder OK nach 3 h 12 min (zuletzt CRIT)`), Erinnerungen `weiter CRIT seit 4 h`. - Recovery-Meldungen sind in der UI schaltbar: globaler Schalter `alerts.recovery_enabled` (Default an) in der Section "Alarmierung" und pro Regel über die Ereignisarten (`kinds` enthält `recovery` oder nicht). Greift zusätzlich zur Regel "Recovery nur, wenn das Problem vorher gemeldet wurde". Ist Recovery aus, bleibt sie im Verlauf sichtbar, wird aber nicht gesendet. - Verzögerung (Phase 6b): Section "Alarmierung" bekommt `alerts.default_delay_minutes` (Werkseinstellung 15; ersetzt `poll.min_duration`). Jede Regel kann die Verzögerung überschreiben, auch mit 0. Passen mehrere Regeln, gilt die kleinste. Sie zählt ab der letzten Statusänderung, eine Verschlechterung startet sie neu; wird das Problem vorher OK, gibt es weder Alarm noch Recovery. Das Dashboard zeigt wartende Probleme ("wartet noch 8 min"). Mute und Downtime wirken auch während der Wartezeit. Host-Alarme und der Checkmk-Ausfall (Phase 6c) kommen immer sofort, Host-Verzögerung `alerts.host_delay_minutes` Standard 0. Die vorinstallierte Regel "ZFS Zpool" (`*zpool*`) hat Verzögerung 0, damit gilt ab Werk: Zpool sofort, alles andere nach 15 min. -- Phase 6d (nach 6c): API-Selbsttest. Lesend beim Start, täglich 04:00 und sofort bei Checkmk-Versionswechsel: `/version`, Service- und Host-Abfrage (alle benötigten Spalten in der Antwort), Downtime-Abruf (GET). Ergebnis auf einer UI-Seite, Teilcheck `checkmk_api` in `/health`, Chat-Meldung bei Fehlschlag oder Versionswechsel (und bei Rückkehr zu ok). Unterscheidet "nicht erreichbar" von "Antwort unerwartet". - - Aufgenommen nach Prüfung des Referenzrepos `jlk/checkmk-llm-server` (MIT, nur als Referenz; es belegt weder Host-Live-Status noch die Downtime-Liste): (a) toleranter Antwort-Parser, der `value` oder `members` akzeptiert und `state_type` als int oder String (`hard`/`soft`) zulässt; (b) Downtime-Fallback: scheitert `GET /domain-types/downtime/collections/all`, ermittelt der Bot Downtimes aus `scheduled_downtime_depth > 0` der Service- und Host-Abfrage und meldet das im Selbsttest als Abweichung; (c) lesbare Fehlerhinweise je Status (401, 403, 404, 422, 5xx) statt nur `HTTP `. Die Spalten `last_state_change`, `is_flapping`, `state_type` und der Host-Endpunkt bleiben bis zum Selbsttest gegen 2.5 ungeprüft. - - Zweite Referenz `chexma/vibeMK` (GPL-3.0, nur Wissen, kein Code): Host-Live-Status dort per **GET** `/domain-types/host/collections/all` mit `columns=name,state,acknowledged,scheduled_downtime_depth`, State 0 clientseitig verworfen, Host-State 0/1/2 = UP/DOWN/UNREACH (bestätigt); Downtime-Liste per GET `/domain-types/downtime/collections/all` mit `query` (`host_name`, `is_service`), `downtime_type` als Listenfilter dort nicht belegt; `state_type` int 0 = soft, 1 = hard. Fallback im Selbsttest: scheitert der POST der Host-Abfrage, GET mit `columns` und clientseitigem Filter; scheitert der Downtime-Abruf mit `downtime_type`, GET mit `query` und `is_service`. +- Phase 6d (nach 6c): API-Selbsttest. Lesend beim Start, täglich 04:00 und sofort bei Checkmk-Versionswechsel: `/version`, Service- und Host-Abfrage (alle benötigten Spalten in der Antwort), Downtime-Abruf (GET). Ergebnis auf einer UI-Seite, Teilcheck `checkmk_api` in `/health`, Chat-Meldung bei Fehlschlag oder Versionswechsel (und bei Rückkehr zu ok). Unterscheidet "nicht erreichbar" von "Antwort unerwartet". **Erledigt (Code geschrieben, ungetestet), Details und Annahmen siehe Abschnitt "Phase 6d API-Selbsttest" oben.** + - Aufgenommen nach Prüfung des Referenzrepos `jlk/checkmk-llm-server` (MIT, nur als Referenz; es belegt weder Host-Live-Status noch die Downtime-Liste): (a) toleranter Antwort-Parser, der `value` oder `members` akzeptiert und `state_type` als int oder String (`hard`/`soft`) zulässt; (b) Downtime-Fallback: scheitert `GET /domain-types/downtime/collections/all`, ermittelt der Bot Downtimes aus `scheduled_downtime_depth > 0` der Service- und Host-Abfrage und meldet das im Selbsttest als Abweichung; (c) lesbare Fehlerhinweise je Status (401, 403, 404, 422, 5xx) statt nur `HTTP `. Die Spalten `last_state_change`, `is_flapping`, `state_type` und der Host-Endpunkt bleiben bis zum Selbsttest gegen 2.5 ungeprüft. (umgesetzt: `extract_extensions`, `_is_hard`/`HARD_STATE_TYPES`, `downtimes_from_states`, `status_hint` in `app/checkmk.py`.) + - Zweite Referenz `chexma/vibeMK` (GPL-3.0, nur Wissen, kein Code): Host-Live-Status dort per **GET** `/domain-types/host/collections/all` mit `columns=name,state,acknowledged,scheduled_downtime_depth`, State 0 clientseitig verworfen, Host-State 0/1/2 = UP/DOWN/UNREACH (bestätigt); Downtime-Liste per GET `/domain-types/downtime/collections/all` mit `query` (`host_name`, `is_service`), `downtime_type` als Listenfilter dort nicht belegt; `state_type` int 0 = soft, 1 = hard. Fallback im Selbsttest: scheitert der POST der Host-Abfrage, GET mit `columns` und clientseitigem Filter; scheitert der Downtime-Abruf mit `downtime_type`, GET mit `query` und `is_service`. (umgesetzt: `problem_hosts_with_fallback`/`selftest_hosts`, `downtimes_with_fallback`/`_downtimes_by_query` in `app/checkmk.py`.) - Entschieden nach Phase 6b: Erinnerung pro Regel: leer = Standard, 0 = aus, >0 = Stunden. Keine Recovery ohne vorherigen Alarm bleibt so. Migrationen laufen künftig in einer Transaktion pro Migration mit Rollback. `deploy/first-run.sh` richtet das Dev-venv ein, lässt `pytest` laufen und packt Log, Python-Version, Paketliste und Git-Stand ohne Secrets in ein Archiv. Zugriff auf den Test-LXC per SSH nur dort, eigener Benutzer ohne Root, jeder Befehl mit Freigabe. Nach dem ersten grünen `pytest` und vor der Abnahme: `docs/THREAT_MODEL.md` (STRIDE) und ein einmaliger Sicherheits-Review des gesamten Codes. - Vorgemerkt, noch NICHT entschieden (Phase 10, Idee): Chat mit dem Agenten ("Hermes") in der UI. Heute ist der Agent nur per Telegram/Matrix (`/ask`, `/explain`, Tiefenanalyse) und als Einzelfrage-Testlauf unter `/prompts` erreichbar. Vorschläge zur späteren Entscheidung: nur Admins (Schalter `ui.chat_viewers_allowed`, Standard aus); 6 Runden Kontext, Gespräche mit Gesprächs-ID in `agent_runs`, Nutzer sehen eigene Gespräche, Admins alle unter `/agent`, Aufbewahrung 90 Tage, Rate-Limit 5 pro Minute; Antwort nach Abschluss ohne Streaming ("läuft …" mit Abbrechen, max. 90 s), Tool-Schritte aufklappbar. Das Agentenmodell ist in der Section "Ollama" einstellbar (auch ein Hermes-Modell). Erst nach der Abnahme angehen. - Aktionen: Der Bot führt nie Shell-Befehle aus (weder lokal noch per SSH). Webhook pro Regel (POST JSON an Allowlist-URLs, Audit) kommt erst nach der Abnahme. diff --git a/README.md b/README.md index 5ccf99e..db0b6d3 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,12 @@ Bot für Checkmk. Er liest den Zustand per REST-API, lässt Probleme von einem e - Datenschutz je Backend: Telegram und Matrix haben die Einstellung "Inhalt der Alarmnachrichten": `voll` (Standard) oder `kurz` (Host, Service, Status und Verlaufszeile, ohne Plugin-Output und ohne KI-Text). Für `kurz`-Ziele wird keine KI-Analyse nachgereicht. Eine Überschreibung pro einzelnem Ziel gibt es nicht, weil Ziele in Regeln als freie Zeilen stehen; der Modus gilt je Backend. - Link zur Oberfläche: Mit gesetzter "Basis-URL dieser Oberfläche" (Alarmierung, ohne abschließenden Schrägstrich gespeichert) enthält jede Alarmnachricht einen Link "Details" auf `/alerts#alert-` (Telegram als URL-Button in der Button-Reihe, Matrix als Zeile im Text). Lehnt Telegram die URL ab (z. B. localhost), wird die Nachricht ohne Link-Button gesendet. Leer = kein Link. +### API-Selbsttest (Phase 6d) +- Prüft rein lesend, ob Checkmk wie erwartet antwortet: `/version`, Service-Abfrage, Host-Abfrage, Downtime-Abruf. Jeder Teilcheck meldet `ok`, `abweichend` (z. B. optionale Spalte fehlt, Fallback auf GET statt POST oder auf `query` statt `downtime_type`, Downtimes aus `scheduled_downtime_depth` abgeleitet) oder `fehler` (Pflichtspalte fehlt, Checkmk nicht erreichbar oder Antwort unerwartet), nie mit Antwortinhalt im Text. +- Läuft kurz nach dem Start, danach täglich um 04:00 Uhr lokaler Zeit und markiert einen Lauf zusätzlich als Versionswechsel, wenn sich `versions.checkmk` gegenüber dem letzten Lauf geändert hat. Button "Selbsttest jetzt ausführen" (UI `/selftest`, nur Admins, höchstens 1x pro 30 s, Audit-Eintrag `selftest.run`). +- Ergebnis auf `/selftest` (für alle Rollen lesbar): Status je Teilcheck, Checkmk-Version, Zeitpunkt des letzten Laufs. Teilcheck `checkmk_api` in `/health` liest nur den letzten gespeicherten Lauf (kein eigener API-Aufruf); "noch nicht gelaufen" zählt wie "nicht konfiguriert". +- Chat-Meldung (Art `system`, wie die Checkmk-Ausfallmeldung) bei Wechsel auf abweichend/fehler, bei Rückkehr zu ok und bei Versionswechsel, nie bei gleichem Ergebnis zum Vorlauf. Schalter `selftest.enabled` (Zeitplan) und `selftest.notify` (Chat-Meldung) unter "Selbsttest-Einstellungen", beide Standard an. + ### Web-UI - Anmeldung mit lokalen Nutzern (Argon2), optional TOTP. Rollen `admin` und `viewer`. - Einstellungen mit Verbindungstest: Checkmk, Ollama, Telegram, Matrix, Checkmk Downtime. Dazu die Bereiche Alarmierung (Abfrageintervall, Standard-Verzögerung, Host-Verzögerung, Recovery-Schalter, Standard-Erinnerung, Bündelungsfenster, KI-Analysen pro Minute, Sendelimit, Basis-URL, Baseline-Button) und Aufbewahrung. @@ -119,6 +125,7 @@ Web-UI (FastAPI, Jinja2) ---- SQLite (Einstellungen, Verlauf, Audit, Nutzer) | Phase 6: Prompt-Versionierung mit Rollback und Testbereich, Retention | Code geschrieben, ungetestet | | Phase 6b: Regeln (Ausgabefilter, Verzögerung, KI-Hinweis, Erinnerung, laut/leise), Delta, Erinnerungen, Recovery-Schalter, Zpool-Standardregel | Code geschrieben, ungetestet | | Phase 6c: Baseline, Bestätigungen, Host-Alarme, Ausfallmeldung, Sendelimit, Datenschutz je Backend, UI-Link; atomare Migrationen; Erinnerung je Regel | Code geschrieben, ungetestet | +| Phase 6d: API-Selbsttest (Version, Service-/Host-Abfrage, Downtime-Abruf, Fallbacks, `/health`-Teilcheck `checkmk_api`, UI `/selftest`) | Code geschrieben, ungetestet | | Phase 7: Deployment (`setup-lxc.sh`, Caddy, systemd, Backup, Checkmk-Local-Check) | Dateien geschrieben, nicht auf einem System ausprobiert | | Phase 8: Integrationstests | geplant | diff --git a/app/alerts/system.py b/app/alerts/system.py index 079b0e5..5ea2054 100644 --- a/app/alerts/system.py +++ b/app/alerts/system.py @@ -10,8 +10,8 @@ from .delivery import AdapterProvider, Deliverer from .extras import AlertActions from .repository import AlertRepository from .sendlimit import SendLimiter -from .system_text import (SERVICE_BASELINE, SERVICE_CHECKMK, SYSTEM_HOST, baseline_message, baseline_plain, - outage_plain, restored_plain, wrap) +from .system_text import (SERVICE_BASELINE, SERVICE_CHECKMK, SERVICE_SELFTEST, SYSTEM_HOST, baseline_message, + baseline_plain, outage_plain, restored_plain, wrap) class SystemNotifier: @@ -39,6 +39,9 @@ class SystemNotifier: self._on_baseline([row.key for row in rows]) await self._post(SERVICE_BASELINE, STATE_OK, baseline_plain(rows), baseline_message(rows)) + async def selftest(self, ok: bool, plain: str) -> None: + await self._post(SERVICE_SELFTEST, STATE_OK if ok else STATE_CRIT, plain) + async def _post(self, service: str, state: int, plain: str, message: str | None = None) -> None: event = Event(ts=int(self._clock()), host=SYSTEM_HOST, service=service, kind=KIND_SYSTEM, state=state, output=plain, ai_enabled=False) diff --git a/app/alerts/system_text.py b/app/alerts/system_text.py index fc8bf74..ffe7143 100644 --- a/app/alerts/system_text.py +++ b/app/alerts/system_text.py @@ -12,6 +12,7 @@ SYSTEM_HOST = "checkbot" SERVICE_CHECKMK = "Checkmk-Verbindung" SERVICE_BASELINE = "Baseline" SERVICE_SEND_LIMIT = "Sendelimit" +SERVICE_SELFTEST = "Selbsttest" TOP_COUNT = 10 BULLET = "•" diff --git a/app/checkmk.py b/app/checkmk.py index 33ab364..4399a30 100644 --- a/app/checkmk.py +++ b/app/checkmk.py @@ -1,4 +1,6 @@ +import json from dataclasses import dataclass +from typing import Sequence import httpx @@ -47,14 +49,36 @@ FAILURE_UNEXPECTED = "unexpected" NOT_OK_QUERY = {"op": "!=", "left": "state", "right": str(STATE_OK)} +HOST_QUERY_POST = "post" +HOST_QUERY_GET = "get" + +DOWNTIME_MODE_TYPE = "downtime_type" +DOWNTIME_MODE_QUERY = "query" + +# Lesbare Hinweise je HTTP-Status, nur für Menschen (nie Teil von .brief, das bleibt wortgleich wie bisher). +STATUS_HINTS = { + 401: "Zugangsdaten prüfen", + 403: "Rechte des Automation-Users prüfen", + 404: "Pfad oder Site prüfen", + 422: "Anfrage von Checkmk abgelehnt (Format/Spalten prüfen)", +} +SERVER_ERROR_HINT = "Checkmk-Serverfehler" + + +def status_hint(status: int) -> str: + if status >= 500: + return SERVER_ERROR_HINT + return STATUS_HINTS.get(status, "") + class CheckmkError(Exception): kind = FAILURE_UNEXPECTED - def __init__(self, message: str, brief: str = ""): + def __init__(self, message: str, brief: str = "", hint: str = ""): super().__init__(message) # Kurztext ohne Antwortinhalt, damit er gefahrlos in Chat-Meldungen steht. self.brief = brief or type(self).__name__ + self.hint = hint class CheckmkUnreachable(CheckmkError): @@ -113,8 +137,11 @@ def _equals(column: str, value: str) -> dict: return {"op": "=", "left": column, "right": value} +HARD_STATE_TYPES = frozenset({"1", "hard"}) + + def _is_hard(value) -> bool: - return str(value).lower() in ("1", "hard") + return str(value).lower() in HARD_STATE_TYPES def _parse_service(extensions: dict) -> ServiceState: @@ -142,14 +169,26 @@ def _parse_host(extensions: dict) -> HostState: ) +def extract_extensions(body: dict) -> list[dict]: + # Checkmk nennt die Collection je Endpunkt mal "value", mal "members". + rows = body.get("value", body.get("members", [])) + return [item["extensions"] for item in rows] + + def _items(body, parser) -> list: try: - return [parser(item["extensions"]) for item in body.get("value", [])] + return [parser(extensions) for extensions in extract_extensions(body)] except (KeyError, TypeError, ValueError, AttributeError) as exc: raise CheckmkError(f"Unerwartetes Antwortformat ({type(exc).__name__})", brief="Antwortformat unerwartet") from exc +def downtimes_from_states(services: Sequence[ServiceState], hosts: Sequence[HostState]) -> list[Downtime]: + host_downtimes = [Downtime(h.host, None, "", "", "", "") for h in hosts if h.in_downtime] + service_downtimes = [Downtime(s.host, s.service, "", "", "", "") for s in services if s.in_downtime] + return host_downtimes + service_downtimes + + def is_configured(store) -> bool: return all(store.get(key) for key in (CHECKMK_URL, CHECKMK_SITE, CHECKMK_USER, CHECKMK_SECRET)) @@ -165,6 +204,8 @@ class CheckmkClient: timeout=timeout, transport=transport, ) + # Merkt sich den zuletzt funktionierenden Weg der Host-Abfrage (siehe problem_hosts_with_fallback). + self._host_query_mode = HOST_QUERY_POST @classmethod def from_settings(cls, store) -> "CheckmkClient | None": @@ -186,7 +227,8 @@ class CheckmkClient: status = response.status_code if status >= 400: error = CheckmkUnreachable if status >= 500 else CheckmkError - raise error(f"HTTP {status} on {path}: {response.text[:200]}", brief=f"HTTP {status}") + raise error(f"HTTP {status} on {path}: {response.text[:200]}", brief=f"HTTP {status}", + hint=status_hint(status)) try: return response.json() except ValueError as exc: @@ -203,13 +245,58 @@ class CheckmkClient: ) return _items(body, _parse_service) + async def _fetch_hosts_post(self) -> dict: + return await self._request("POST", HOSTS_PATH, json={"query": NOT_OK_QUERY, "columns": HOST_COLUMNS}) + + async def _fetch_hosts_get(self) -> dict: + return await self._request("GET", HOSTS_PATH, params={"columns": ",".join(HOST_COLUMNS)}) + async def problem_hosts(self) -> list[HostState]: - body = await self._request( - "POST", - HOSTS_PATH, - json={"query": NOT_OK_QUERY, "columns": HOST_COLUMNS}, - ) - return _items(body, _parse_host) + if self._host_query_mode == HOST_QUERY_GET: + hosts = _items(await self._fetch_hosts_get(), _parse_host) + return [h for h in hosts if h.state != STATE_OK] + return _items(await self._fetch_hosts_post(), _parse_host) + + async def _scan_hosts(self) -> tuple[list[dict], list[HostState], str]: + order = (self._host_query_mode, + HOST_QUERY_GET if self._host_query_mode == HOST_QUERY_POST else HOST_QUERY_POST) + last_exc: CheckmkError | None = None + for mode in order: + try: + body = await (self._fetch_hosts_get() if mode == HOST_QUERY_GET else self._fetch_hosts_post()) + except CheckmkError as exc: + last_exc = exc + continue + rows = extract_extensions(body) + try: + hosts = _items(body, _parse_host) + except CheckmkError: + hosts = [] + if mode == HOST_QUERY_GET: + rows = [r for r in rows if str(r.get("state", "")) != "0"] + hosts = [h for h in hosts if h.state != STATE_OK] + self._host_query_mode = mode + return rows, hosts, mode + raise last_exc + + async def problem_hosts_with_fallback(self) -> tuple[list[HostState], str]: + _, hosts, mode = await self._scan_hosts() + return hosts, mode + + async def selftest_hosts(self) -> tuple[list[dict], list[HostState], str]: + return await self._scan_hosts() + + async def raw_services(self) -> dict: + return await self._request("POST", SERVICES_PATH, json={"query": NOT_OK_QUERY, "columns": COLUMNS}) + + async def selftest_services(self) -> tuple[list[dict], list[ServiceState]]: + body = await self.raw_services() + rows = extract_extensions(body) + try: + services = _items(body, _parse_service) + except CheckmkError: + services = [] + return rows, services async def problems(self) -> list[ServiceState]: return await self.problem_services() @@ -236,5 +323,24 @@ class CheckmkClient: downtimes = [d for d in downtimes if d.host.casefold() == wanted] return downtimes + async def _downtimes_by_query(self, host: str | None = None) -> list[Downtime]: + downtimes: list[Downtime] = [] + for is_service in ("0", "1"): + query: dict = {"op": "=", "left": "is_service", "right": is_service} + if host: + query = {"op": "and", "expr": [query, _equals("host_name", host)]} + body = await self._request("GET", DOWNTIMES_PATH, params={"query": json.dumps(query)}) + downtimes += _items(body, _parse_downtime) + if host: + wanted = host.casefold() + downtimes = [d for d in downtimes if d.host.casefold() == wanted] + return downtimes + + async def downtimes_with_fallback(self, host: str | None = None) -> tuple[list[Downtime], str]: + try: + return await self.downtimes(host), DOWNTIME_MODE_TYPE + except CheckmkError: + return await self._downtimes_by_query(host), DOWNTIME_MODE_QUERY + async def aclose(self) -> None: await self._client.aclose() diff --git a/app/constants.py b/app/constants.py index 43eaab6..615e720 100644 --- a/app/constants.py +++ b/app/constants.py @@ -77,6 +77,9 @@ CHECKMK_DT_MAX_HOURS = "checkmk_dt.max_hours" DEFAULT_DT_MAX_HOURS = 24 DT_MAX_HOURS_CEILING = 168 +SELFTEST_ENABLED = "selftest.enabled" +SELFTEST_NOTIFY = "selftest.notify" + RETENTION_ALERTS_DAYS = "retention.alerts_days" RETENTION_AGENT_RUNS_DAYS = "retention.agent_runs_days" RETENTION_STATE_LOG_DAYS = "retention.state_log_days" diff --git a/app/health.py b/app/health.py index 0077eae..210488e 100644 --- a/app/health.py +++ b/app/health.py @@ -7,6 +7,7 @@ from .checkmk import CheckmkClient, is_configured from .constants import DEFAULT_POLL_INTERVAL, FLAG_ON, HEALTH_PING, MATRIX_ENABLED, TELEGRAM_ENABLED from .ollama import OllamaClient from .poller import PollerStatus, read_poll_options +from .selftest import STATUS_OK as SELFTEST_STATUS_OK, SelftestRepository CHECK_TIMEOUT = 10.0 POLLER_STALE_FACTOR = 3 @@ -23,8 +24,8 @@ async def run_checks(checks: dict[str, Check]) -> dict: try: detail = await asyncio.wait_for(check(), CHECK_TIMEOUT) return {"ok": True, "detail": detail} - except NotConfigured: - return {"ok": False, "detail": "nicht konfiguriert"} + except NotConfigured as exc: + return {"ok": False, "detail": str(exc) or "nicht konfiguriert"} except asyncio.TimeoutError: return {"ok": False, "detail": "Timeout"} except Exception as exc: @@ -36,7 +37,8 @@ async def run_checks(checks: dict[str, Check]) -> dict: return {"ok": all(r["ok"] for r in results), "checks": by_name} -def build_checks(store, poller_status: PollerStatus | None = None) -> dict[str, Check]: +def build_checks(store, poller_status: PollerStatus | None = None, + selftest_repo: SelftestRepository | None = None) -> dict[str, Check]: async def db() -> str: store.get(HEALTH_PING) return "ok" @@ -70,6 +72,17 @@ def build_checks(store, poller_status: PollerStatus | None = None) -> dict[str, raise RuntimeError(f"letzter erfolgreicher Zyklus vor {int(age)} s{reason}") return f"letzter erfolgreicher Zyklus vor {int(age)} s" + async def checkmk_api() -> str: + if selftest_repo is None or not is_configured(store): + raise NotConfigured + run = await asyncio.to_thread(selftest_repo.last_run) + if run is None: + raise NotConfigured("noch nicht gelaufen") + if run.overall_status != SELFTEST_STATUS_OK: + failing = ", ".join(f"{c.name}: {c.text}" for c in run.checks if c.status != SELFTEST_STATUS_OK) + raise RuntimeError(f"Selbsttest {run.overall_status}: {failing}"[:200]) + return f"Selbsttest ok, Checkmk {run.version or '?'}" + async def telegram() -> str: adapter = build_telegram(store) if adapter is None: @@ -88,7 +101,7 @@ def build_checks(store, poller_status: PollerStatus | None = None) -> dict[str, finally: await adapter.aclose() - checks = {"db": db, "checkmk": checkmk, "ollama": ollama, "poller": poller} + checks = {"db": db, "checkmk": checkmk, "ollama": ollama, "poller": poller, "checkmk_api": checkmk_api} if store.get(TELEGRAM_ENABLED) == FLAG_ON: checks["telegram"] = telegram if store.get(MATRIX_ENABLED) == FLAG_ON: diff --git a/app/main.py b/app/main.py index ac0b5c8..47886f6 100644 --- a/app/main.py +++ b/app/main.py @@ -18,7 +18,8 @@ from .runtime import Runtime from .settings import SettingsStore from .users import UserStore from .web import (account, agent_pages, alerts, auth, baseline_pages, matrix_section, mutes_pages, pages, - prompts_pages, prompts_test_pages, retention_pages, routing_pages, telegram_section) + prompts_pages, prompts_test_pages, retention_pages, routing_pages, selftest_pages, + telegram_section) from .web.deps import LoginRequired, redirect, templates, verify_csrf STATIC_DIR = Path(__file__).parent / "web" / "static" @@ -75,7 +76,7 @@ def create_app(config: FileConfig | None = None) -> FastAPI: @app.get("/health") async def health() -> JSONResponse: - result = await run_checks(build_checks(app.state.store, runtime.status)) + result = await run_checks(build_checks(app.state.store, runtime.status, runtime.selftest_repo)) return JSONResponse(result, status_code=200 if result["ok"] else 503) app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") @@ -91,5 +92,6 @@ def create_app(config: FileConfig | None = None) -> FastAPI: app.include_router(prompts_pages.router) app.include_router(prompts_test_pages.router) app.include_router(retention_pages.router) + app.include_router(selftest_pages.router) app.include_router(pages.router) return app diff --git a/app/migrations.py b/app/migrations.py index e2aa9a5..8d1ee8c 100644 --- a/app/migrations.py +++ b/app/migrations.py @@ -171,4 +171,12 @@ MIGRATIONS = [ """ UPDATE routes SET reminder_hours = NULL WHERE reminder_hours = 0; """, + """ + CREATE TABLE selftest_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + ts INTEGER NOT NULL, + version TEXT, + checks TEXT NOT NULL + ); + """, ] diff --git a/app/runtime.py b/app/runtime.py index 304ecb4..e04f026 100644 --- a/app/runtime.py +++ b/app/runtime.py @@ -11,7 +11,8 @@ from .alerts import (AlertActions, AlertAnalyzer, AlertPipeline, AlertRepository from .chat import AdapterRegistry from .chatops import ChatOps from .constants import (ALERTS_BUNDLE_SECONDS, ALERTS_RECOVERY_ENABLED, ALERTS_REMINDER_HOURS, DEFAULT_BUNDLE_SECONDS, - DEFAULT_REMINDER_HOURS, FLAG_OFF, FLAG_ON, MAX_REMINDER_HOURS) + DEFAULT_REMINDER_HOURS, FLAG_OFF, FLAG_ON, MAX_REMINDER_HOURS, SELFTEST_ENABLED, + SELFTEST_NOTIFY) from .db import Database from .agent_runs import configured_secrets from .mutes import MuteService @@ -22,6 +23,7 @@ from .prompts import (PROMPT_AGENT, PROMPT_ALERT, PromptRepository, PromptServic from .retention import RetentionRepository, RetentionRunner, RetentionService, policy_from from .reminders import ReminderWorker from .routing import RouteService, seed_default_rules +from .selftest import SelftestRepository, SelftestRunner, SelftestService class Runtime: @@ -54,6 +56,13 @@ class Runtime: on_baseline=self.pipeline.reset_reported) self.poller = Poller(store, self.state, self.pipeline.handle, self.status, rules=self.routes.enabled_routes, notifier=self.notifier, baseline=self.baseline) + self.selftest_repo = SelftestRepository(db) + self.selftest = SelftestService(store, self.selftest_repo) + self.selftest_runner = SelftestRunner( + db, self.selftest, self.notifier, + enabled=lambda: store.get(SELFTEST_ENABLED, FLAG_ON) != FLAG_OFF, + notify_enabled=lambda: store.get(SELFTEST_NOTIFY, FLAG_ON) != FLAG_OFF, + ) self.reminders = ReminderWorker( self.state.load, self.routes.enabled_routes, self.mutes.active, lambda: min(store.get_int(ALERTS_REMINDER_HOURS, DEFAULT_REMINDER_HOURS), MAX_REMINDER_HOURS), @@ -73,6 +82,7 @@ class Runtime: pipeline_task = asyncio.create_task(self.pipeline.run()) listener_tasks = [asyncio.create_task(run()) for run in self.chatops.listener_runners()] retention_task = asyncio.create_task(self.retention_runner.run()) + selftest_task = asyncio.create_task(self.selftest_runner.run()) try: yield finally: @@ -81,6 +91,8 @@ class Runtime: await asyncio.gather(*listener_tasks, return_exceptions=True) retention_task.cancel() await asyncio.gather(retention_task, return_exceptions=True) + selftest_task.cancel() + await asyncio.gather(selftest_task, return_exceptions=True) poller_task.cancel() reminder_task.cancel() await asyncio.gather(poller_task, reminder_task, return_exceptions=True) diff --git a/app/selftest/__init__.py b/app/selftest/__init__.py new file mode 100644 index 0000000..177db47 --- /dev/null +++ b/app/selftest/__init__.py @@ -0,0 +1,15 @@ +from .checks import check_downtimes, check_hosts, check_services, check_version +from .constants import (AUDIT_RUN, AUDIT_TARGET, CHECK_DOWNTIMES, CHECK_HOSTS, CHECK_LABELS, CHECK_NAMES, + CHECK_SERVICES, CHECK_VERSION, STATUS_DEVIATED, STATUS_FAILED, STATUS_OK) +from .model import CheckResult, SelftestRun +from .notify import message_for +from .repository import SelftestRepository +from .runner import SelftestRunner, should_run +from .service import SelftestNotConfigured, SelftestService + +__all__ = [ + "AUDIT_RUN", "AUDIT_TARGET", "CHECK_DOWNTIMES", "CHECK_HOSTS", "CHECK_LABELS", "CHECK_NAMES", + "CHECK_SERVICES", "CHECK_VERSION", "CheckResult", "STATUS_DEVIATED", "STATUS_FAILED", "STATUS_OK", + "SelftestNotConfigured", "SelftestRepository", "SelftestRun", "SelftestRunner", "SelftestService", + "check_downtimes", "check_hosts", "check_services", "check_version", "message_for", "should_run", +] diff --git a/app/selftest/checks.py b/app/selftest/checks.py new file mode 100644 index 0000000..5da0530 --- /dev/null +++ b/app/selftest/checks.py @@ -0,0 +1,92 @@ +from typing import Sequence + +from ..checkmk import (COLUMNS, DOWNTIME_MODE_QUERY, HOST_COLUMNS, HOST_QUERY_GET, CheckmkClient, CheckmkError, + CheckmkUnreachable, HostState, ServiceState, downtimes_from_states) +from .constants import (CHECK_DOWNTIMES, CHECK_HOSTS, CHECK_SERVICES, CHECK_VERSION, STATUS_DEVIATED, + STATUS_FAILED, STATUS_OK) +from .model import CheckResult + +SERVICE_REQUIRED = ("host_name", "description", "state") +SERVICE_OPTIONAL = tuple(c for c in COLUMNS if c not in SERVICE_REQUIRED) +HOST_REQUIRED = ("name", "state") +HOST_OPTIONAL = tuple(c for c in HOST_COLUMNS if c not in HOST_REQUIRED) + + +def _missing_columns(rows: Sequence[dict], required: Sequence[str], + optional: Sequence[str]) -> tuple[list[str], list[str]]: + missing_required: set[str] = set() + missing_optional: set[str] = set() + for row in rows: + missing_required |= {c for c in required if c not in row} + missing_optional |= {c for c in optional if c not in row} + return sorted(missing_required), sorted(missing_optional) + + +def _unreachable_or_unexpected(name: str, exc: CheckmkError) -> CheckResult: + if isinstance(exc, CheckmkUnreachable): + return CheckResult(name, STATUS_FAILED, f"nicht erreichbar ({exc.brief})") + return CheckResult(name, STATUS_FAILED, f"Antwort unerwartet ({exc.brief})") + + +async def check_version(client: CheckmkClient) -> tuple[CheckResult, str]: + try: + info = await client.version() + except CheckmkError as exc: + return _unreachable_or_unexpected(CHECK_VERSION, exc), "" + versions = info.get("versions") + version = versions.get("checkmk") if isinstance(versions, dict) else None + if not version: + return CheckResult(CHECK_VERSION, STATUS_DEVIATED, "Feld versions.checkmk fehlt in der Antwort"), "" + return CheckResult(CHECK_VERSION, STATUS_OK, f"Checkmk {version}"), str(version) + + +async def check_services(client: CheckmkClient) -> tuple[CheckResult, list[ServiceState]]: + try: + rows, services = await client.selftest_services() + except CheckmkError as exc: + return _unreachable_or_unexpected(CHECK_SERVICES, exc), [] + missing_required, missing_optional = _missing_columns(rows, SERVICE_REQUIRED, SERVICE_OPTIONAL) + if missing_required: + return CheckResult(CHECK_SERVICES, STATUS_FAILED, + f"Pflichtspalte(n) fehlen: {', '.join(missing_required)}"), services + if missing_optional: + return CheckResult(CHECK_SERVICES, STATUS_DEVIATED, + f"optionale Spalte(n) fehlen, Fallback genutzt: {', '.join(missing_optional)}"), services + return CheckResult(CHECK_SERVICES, STATUS_OK, + f"{len(services)} Probleme gelesen, alle Spalten vorhanden"), services + + +async def check_hosts(client: CheckmkClient) -> tuple[CheckResult, list[HostState]]: + try: + rows, hosts, mode = await client.selftest_hosts() + except CheckmkError as exc: + return _unreachable_or_unexpected(CHECK_HOSTS, exc), [] + missing_required, missing_optional = _missing_columns(rows, HOST_REQUIRED, HOST_OPTIONAL) + if missing_required: + return CheckResult(CHECK_HOSTS, STATUS_FAILED, + f"Pflichtspalte(n) fehlen: {', '.join(missing_required)}"), hosts + notes = [] + if mode == HOST_QUERY_GET: + notes.append("GET-Fallback verwendet, POST wurde abgelehnt") + if missing_optional: + notes.append(f"optionale Spalte(n) fehlen, Fallback genutzt: {', '.join(missing_optional)}") + if notes: + return CheckResult(CHECK_HOSTS, STATUS_DEVIATED, "; ".join(notes)), hosts + return CheckResult(CHECK_HOSTS, STATUS_OK, f"{len(hosts)} Probleme gelesen, alle Spalten vorhanden"), hosts + + +async def check_downtimes(client: CheckmkClient, services: Sequence[ServiceState], + hosts: Sequence[HostState]) -> CheckResult: + try: + downtimes, mode = await client.downtimes_with_fallback() + except CheckmkUnreachable as exc: + return CheckResult(CHECK_DOWNTIMES, STATUS_FAILED, f"nicht erreichbar ({exc.brief})") + except CheckmkError as exc: + derived = downtimes_from_states(services, hosts) + return CheckResult(CHECK_DOWNTIMES, STATUS_DEVIATED, + f"Downtime-Endpunkt abgelehnt ({exc.brief}), Fallback über " + f"scheduled_downtime_depth: {len(derived)} Downtime(s) abgeleitet") + if mode == DOWNTIME_MODE_QUERY: + return CheckResult(CHECK_DOWNTIMES, STATUS_DEVIATED, + f"downtime_type abgelehnt, Fallback über query verwendet, {len(downtimes)} gelesen") + return CheckResult(CHECK_DOWNTIMES, STATUS_OK, f"{len(downtimes)} Downtime(s) gelesen") diff --git a/app/selftest/constants.py b/app/selftest/constants.py new file mode 100644 index 0000000..a97b82e --- /dev/null +++ b/app/selftest/constants.py @@ -0,0 +1,25 @@ +STATUS_OK = "ok" +STATUS_DEVIATED = "abweichend" +STATUS_FAILED = "fehler" +STATUSES = (STATUS_OK, STATUS_DEVIATED, STATUS_FAILED) + +CHECK_VERSION = "version" +CHECK_SERVICES = "services" +CHECK_HOSTS = "hosts" +CHECK_DOWNTIMES = "downtimes" +CHECK_NAMES = (CHECK_VERSION, CHECK_SERVICES, CHECK_HOSTS, CHECK_DOWNTIMES) +CHECK_LABELS = { + CHECK_VERSION: "Version", + CHECK_SERVICES: "Service-Abfrage", + CHECK_HOSTS: "Host-Abfrage", + CHECK_DOWNTIMES: "Downtime-Abruf", +} + +RUN_HOUR = 4 +START_DELAY = 20.0 +CHECK_INTERVAL = 900.0 +MANUAL_COOLDOWN_SECONDS = 30.0 + +AUDIT_RUN = "selftest.run" +AUDIT_TARGET = "selftest" +SYSTEM_USER = "system" diff --git a/app/selftest/model.py b/app/selftest/model.py new file mode 100644 index 0000000..b249059 --- /dev/null +++ b/app/selftest/model.py @@ -0,0 +1,31 @@ +from dataclasses import dataclass, field + +from .constants import STATUS_DEVIATED, STATUS_FAILED, STATUS_OK + + +@dataclass(frozen=True) +class CheckResult: + name: str + status: str + text: str + + +@dataclass(frozen=True) +class SelftestRun: + ts: int + version: str + checks: list[CheckResult] = field(default_factory=list) + version_changed: bool = False + + @property + def ok(self) -> bool: + return self.overall_status == STATUS_OK + + @property + def overall_status(self) -> str: + statuses = {c.status for c in self.checks} + if STATUS_FAILED in statuses: + return STATUS_FAILED + if STATUS_DEVIATED in statuses: + return STATUS_DEVIATED + return STATUS_OK diff --git a/app/selftest/notify.py b/app/selftest/notify.py new file mode 100644 index 0000000..0ee9a8e --- /dev/null +++ b/app/selftest/notify.py @@ -0,0 +1,20 @@ +from .constants import STATUS_OK +from .model import SelftestRun + + +def _failing_text(run: SelftestRun) -> str: + return ", ".join(f"{c.name}: {c.text}" for c in run.checks if c.status != STATUS_OK) + + +def message_for(previous: SelftestRun | None, current: SelftestRun) -> str | None: + if previous is None: + return None + if current.version_changed: + return (f"Checkmk-Version von {previous.version or '?'} auf {current.version or '?'}, " + f"Selbsttest: {current.overall_status}") + prev_status, cur_status = previous.overall_status, current.overall_status + if prev_status == cur_status: + return None + if cur_status == STATUS_OK: + return "Selbsttest wieder ok" + return f"Selbsttest {cur_status}: {_failing_text(current)}" diff --git a/app/selftest/repository.py b/app/selftest/repository.py new file mode 100644 index 0000000..6bc40c7 --- /dev/null +++ b/app/selftest/repository.py @@ -0,0 +1,25 @@ +import json + +from ..db import Database +from .model import CheckResult, SelftestRun + + +class SelftestRepository: + def __init__(self, db: Database): + self._db = db + + def last_run(self) -> SelftestRun | None: + rows = self._db.query("SELECT ts, version, checks FROM selftest_state WHERE id=1") + if not rows: + return None + row = rows[0] + checks = [CheckResult(**item) for item in json.loads(row["checks"])] + return SelftestRun(row["ts"], row["version"] or "", checks) + + def save_run(self, run: SelftestRun) -> None: + payload = json.dumps([{"name": c.name, "status": c.status, "text": c.text} for c in run.checks]) + self._db.execute( + "INSERT INTO selftest_state(id, ts, version, checks) VALUES (1,?,?,?) " + "ON CONFLICT(id) DO UPDATE SET ts=excluded.ts, version=excluded.version, checks=excluded.checks", + (run.ts, run.version, payload), + ) diff --git a/app/selftest/runner.py b/app/selftest/runner.py new file mode 100644 index 0000000..33dd8b7 --- /dev/null +++ b/app/selftest/runner.py @@ -0,0 +1,88 @@ +import asyncio +import logging +import time +from datetime import datetime +from typing import Awaitable, Callable + +from .. import audit +from ..alerts import SlidingWindowLimiter, SystemNotifier +from ..db import Database +from .constants import (AUDIT_RUN, AUDIT_TARGET, CHECK_INTERVAL, MANUAL_COOLDOWN_SECONDS, RUN_HOUR, START_DELAY, + SYSTEM_USER) +from .model import SelftestRun +from .notify import message_for +from .service import SelftestNotConfigured, SelftestService + +log = logging.getLogger(__name__) + + +def should_run(now: datetime, last_run_ts: float | None, startup: bool) -> bool: + if last_run_ts is not None and datetime.fromtimestamp(last_run_ts).date() == now.date(): + return False + return startup or now.hour >= RUN_HOUR + + +class SelftestRunner: + def __init__(self, db: Database, service: SelftestService, notifier: SystemNotifier, + enabled: Callable[[], bool], notify_enabled: Callable[[], bool], + clock: Callable[[], datetime] = datetime.now, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, start_delay: float = START_DELAY, + interval: float = CHECK_INTERVAL, cooldown: float = MANUAL_COOLDOWN_SECONDS): + self._db = db + self._service = service + self._notifier = notifier + self._enabled = enabled + self._notify_enabled = notify_enabled + self._clock = clock + self._sleep = sleep + self._start_delay = start_delay + self._interval = interval + self._manual_limiter = SlidingWindowLimiter(time.time, cooldown) + self._last_attempt: float | None = None + + def try_manual(self) -> bool: + return self._manual_limiter.try_acquire(1) + + async def _execute(self, user: str) -> SelftestRun | None: + previous = await asyncio.to_thread(self._service.last_run) + run = await self._service.run() + await asyncio.to_thread(audit.record, self._db, user, AUDIT_RUN, AUDIT_TARGET) + if self._notify_enabled(): + text = message_for(previous, run) + if text: + await self._notifier.selftest(run.ok, text) + return run + + async def run_manual(self, user: str) -> SelftestRun | None: + try: + return await self._execute(user) + except SelftestNotConfigured: + return None + + async def tick(self, startup: bool) -> bool: + if not self._enabled(): + return False + record = await asyncio.to_thread(self._service.last_run) + known = [ts for ts in (record.ts if record else None, self._last_attempt) if ts is not None] + now = self._clock() + if not should_run(now, max(known) if known else None, startup): + return False + self._last_attempt = now.timestamp() + try: + await self._execute(SYSTEM_USER) + except SelftestNotConfigured: + pass + except Exception: + log.exception("Selbsttest fehlgeschlagen") + return True + + async def run(self) -> None: + await self._sleep(self._start_delay) + startup = True + while True: + try: + await self.tick(startup) + except Exception: + log.exception("Selbsttest: Prüfung fehlgeschlagen") + startup = False + await self._sleep(self._interval) diff --git a/app/selftest/service.py b/app/selftest/service.py new file mode 100644 index 0000000..d9d3791 --- /dev/null +++ b/app/selftest/service.py @@ -0,0 +1,43 @@ +import asyncio +import time +from typing import Callable + +from ..checkmk import CheckmkClient +from .checks import check_downtimes, check_hosts, check_services, check_version +from .model import SelftestRun +from .repository import SelftestRepository + + +class SelftestNotConfigured(Exception): + pass + + +class SelftestService: + def __init__(self, store, repo: SelftestRepository, + client_factory: Callable[[object], CheckmkClient | None] = CheckmkClient.from_settings, + clock: Callable[[], float] = time.time): + self._store = store + self._repo = repo + self._client_factory = client_factory + self._clock = clock + + def last_run(self) -> SelftestRun | None: + return self._repo.last_run() + + async def run(self) -> SelftestRun: + client = self._client_factory(self._store) + if client is None: + raise SelftestNotConfigured + try: + version_check, version = await check_version(client) + services_check, services = await check_services(client) + hosts_check, hosts = await check_hosts(client) + downtimes_check = await check_downtimes(client, services, hosts) + finally: + await client.aclose() + previous = await asyncio.to_thread(self._repo.last_run) + version_changed = bool(previous and previous.version and version and previous.version != version) + run = SelftestRun(int(self._clock()), version, + [version_check, services_check, hosts_check, downtimes_check], version_changed) + await asyncio.to_thread(self._repo.save_run, run) + return run diff --git a/app/web/pages.py b/app/web/pages.py index 7e10f60..d167239 100644 --- a/app/web/pages.py +++ b/app/web/pages.py @@ -14,7 +14,8 @@ from ..constants import (CHECKMK_CA_FILE, CHECKMK_SECRET, CHECKMK_SITE, CHECKMK_ from ..health import build_checks, run_checks from ..ollama import OllamaClient from ..users import MIN_PASSWORD_LEN, ROLES, User, UserError -from . import alerting_section, downtime_section, matrix_section, retention_section, telegram_section +from . import (alerting_section, downtime_section, matrix_section, retention_section, selftest_section, + telegram_section) from .deps import (FLASH_ERROR, FLASH_OK, current_user, flash, redirect, render, require_admin) from .problem_status import problems_with_status from .sections import (URL_RE, Field, Section, check, collect, effective, find_section, form_values, @@ -124,6 +125,7 @@ SECTIONS = { alerting_section.SLUG: alerting_section.SECTION, downtime_section.SLUG: downtime_section.SECTION, retention_section.SLUG: retention_section.SECTION, + selftest_section.SLUG: selftest_section.SECTION, } @@ -131,7 +133,7 @@ SECTIONS = { async def dashboard(request: Request, user: User = Depends(current_user)): store = request.app.state.store runtime = request.app.state.runtime - result = await run_checks(build_checks(store, runtime.status)) + result = await run_checks(build_checks(store, runtime.status, runtime.selftest_repo)) setup_open = user.is_admin and not all( section_done(store, s) for s in SECTIONS.values() if not s.optional) return render(request, "dashboard.html", user=user, result=result, setup_open=setup_open, diff --git a/app/web/selftest_pages.py b/app/web/selftest_pages.py new file mode 100644 index 0000000..88a1778 --- /dev/null +++ b/app/web/selftest_pages.py @@ -0,0 +1,32 @@ +from fastapi import APIRouter, Depends, Request + +from ..selftest import CHECK_LABELS +from ..users import User +from .deps import FLASH_ERROR, FLASH_OK, current_user, flash, redirect, render, require_admin + +router = APIRouter() + +PATH = "/selftest" +RUN_PATH = PATH + "/run" +MSG_RATE_LIMITED = "Bitte kurz warten (höchstens 1x alle 30 s)." +MSG_NOT_CONFIGURED = "Checkmk ist nicht konfiguriert." + + +@router.get(PATH) +def selftest_page(request: Request, user: User = Depends(current_user)): + run = request.app.state.runtime.selftest.last_run() + return render(request, "selftest.html", user=user, run=run, check_labels=CHECK_LABELS) + + +@router.post(RUN_PATH) +async def selftest_run(request: Request, admin: User = Depends(require_admin)): + runner = request.app.state.runtime.selftest_runner + if not runner.try_manual(): + flash(request, MSG_RATE_LIMITED, FLASH_ERROR) + return redirect(PATH) + run = await runner.run_manual(admin.username) + if run is None: + flash(request, MSG_NOT_CONFIGURED, FLASH_ERROR) + else: + flash(request, f"Selbsttest ausgeführt: {run.overall_status}.", FLASH_OK if run.ok else FLASH_ERROR) + return redirect(PATH) diff --git a/app/web/selftest_section.py b/app/web/selftest_section.py new file mode 100644 index 0000000..ea69f0b --- /dev/null +++ b/app/web/selftest_section.py @@ -0,0 +1,21 @@ +from ..constants import SELFTEST_ENABLED, SELFTEST_NOTIFY +from .sections import Field, Section + +SLUG = "selftest" + + +def validate(values: dict) -> list[str]: + return [] + + +SECTION = Section( + slug=SLUG, + title="Selbsttest", + fields=( + Field(SELFTEST_ENABLED, "Selbsttest aktiv (täglich 04:00 Uhr und beim Start)", kind="checkbox"), + Field(SELFTEST_NOTIFY, "Bei Fehlschlag, Abweichung, Rückkehr zu ok oder Versionswechsel in den Chat " + "melden", kind="checkbox"), + ), + validate=validate, + optional=True, +) diff --git a/app/web/templates/base.html b/app/web/templates/base.html index aef9b56..74a6596 100644 --- a/app/web/templates/base.html +++ b/app/web/templates/base.html @@ -19,6 +19,7 @@ Regeln Stummschaltungen Prompts + Selbsttest {% if user.is_admin %} Setup Checkmk @@ -28,6 +29,7 @@ Alarmierung Downtime Aufbewahrung + Selbsttest-Einstellungen Agent Benutzer Audit diff --git a/app/web/templates/selftest.html b/app/web/templates/selftest.html new file mode 100644 index 0000000..11a5bb6 --- /dev/null +++ b/app/web/templates/selftest.html @@ -0,0 +1,37 @@ +{% extends "base.html" %} +{% block title %}Selbsttest{% endblock %} +{% block content %} +

Selbsttest

+
+

Prüft lesend, ob die Checkmk-REST-API wie erwartet antwortet (Version, Service- und Host-Abfrage, Downtime-Abruf). Läuft beim Start, täglich um 04:00 Uhr und bei einem Versionswechsel, ändert nichts in Checkmk. Fallbacks (z. B. GET statt POST) stehen im Text des jeweiligen Teilchecks.

+ {% if run %} +

Checkmk-Version: {{ run.version or '–' }} · letzter Lauf: {{ run.ts | fmt_ts }} + {{ run.overall_status }} +

+
+ + + + {% for c in run.checks %} + + + + + + {% endfor %} + +
TeilcheckStatusText
{{ check_labels.get(c.name, c.name) }}{{ c.status }}{{ c.text }}
+
+ {% else %} +

Noch kein Lauf.

+ {% endif %} +
+{% if user.is_admin %} +
+
+ {{ csrf_input }} +
+
+
+{% endif %} +{% endblock %} diff --git a/deploy/README.md b/deploy/README.md index 42bc47a..6ab9ca3 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -167,7 +167,7 @@ install -m 0755 /opt/checkbot/deploy/checkmk/checkbot-local-check /usr/lib/check Für ein Intervall von 2 Minuten statt jeder Agent-Abfrage in `/usr/lib/check_mk_agent/local/120/` ablegen. Der Check braucht nur das System-`python3` und ruft `http://127.0.0.1:8080/health` direkt ab (nicht über Caddy). Andere URL: `CHECKBOT_HEALTH_URL`. Timeout 15 s, weil der Bot seine Backends selbst mit bis zu 10 s prüft. -Services: `Checkbot_api`, `Checkbot_db`, `Checkbot_checkmk`, `Checkbot_ollama`, `Checkbot_poller`, `Checkbot_telegram`, `Checkbot_matrix`. +Services: `Checkbot_api`, `Checkbot_db`, `Checkbot_checkmk`, `Checkbot_checkmk_api`, `Checkbot_ollama`, `Checkbot_poller`, `Checkbot_telegram`, `Checkbot_matrix`. Bewertung (Entscheidung): @@ -175,10 +175,12 @@ Bewertung (Entscheidung): | --- | --- | | Teilcheck ok | OK | | Telegram/Matrix fehlen im JSON (in der UI deaktiviert) | OK, "deaktiviert" | -| `nicht konfiguriert` (Checkmk, Ollama, Poller, aktivierter Chat-Adapter) | WARN: der Bot ist ohne Konfiguration nicht voll funktionsfähig, aber nichts ist kaputt. Vor der Einrichtung ist WARN erwartbar | -| Fehler, Timeout, Poller-Zyklus überfällig | CRIT | +| `nicht konfiguriert` (Checkmk, Ollama, Poller, aktivierter Chat-Adapter, `checkmk_api` ohne Checkmk-Konfiguration oder "noch nicht gelaufen") | WARN: der Bot ist ohne Konfiguration nicht voll funktionsfähig, aber nichts ist kaputt. Vor der Einrichtung ist WARN erwartbar | +| Fehler, Timeout, Poller-Zyklus überfällig, Selbsttest `abweichend` oder `fehler` (`checkmk_api`) | CRIT | | Bot nicht erreichbar oder ungültige Antwort | CRIT für alle Services | +`Checkbot_checkmk_api` liest nur den zuletzt gespeicherten Selbsttest-Lauf (siehe UI `/selftest`, Phase 6d), macht selbst keinen eigenen API-Aufruf. "noch nicht gelaufen" (kurz nach dem Start) zählt wie "nicht konfiguriert" als WARN. + **Variante B: Active Check "HTTP"** auf `https:///health`, erwarteter Status 200. Der Server braucht Zugriff (Monitoring-IP in `CHECKBOT_MONITOR_IP`) und muss der Caddy-CA vertrauen, sonst Zertifikatsprüfung ausschalten. Achtung: `/health` liefert 503, solange irgendein Teilcheck nicht ok ist, auch "nicht konfiguriert". Ein WARN/CRIT ist vor der vollständigen Einrichtung also normal, und die Detailtexte bekommt man nur im Body. ## 7. Firewall diff --git a/deploy/checkmk/checkbot-local-check b/deploy/checkmk/checkbot-local-check index 2db9711..d04072a 100755 --- a/deploy/checkmk/checkbot-local-check +++ b/deploy/checkmk/checkbot-local-check @@ -9,10 +9,12 @@ import urllib.request HEALTH_URL = os.environ.get("CHECKBOT_HEALTH_URL", "http://127.0.0.1:8080/health") # Der Bot prüft seine Backends mit bis zu 10 s Timeout, daher etwas mehr. TIMEOUT_SECONDS = 15 -CHECK_NAMES = ("db", "checkmk", "ollama", "poller", "telegram", "matrix") +CHECK_NAMES = ("db", "checkmk", "checkmk_api", "ollama", "poller", "telegram", "matrix") SERVICE_PREFIX = "Checkbot_" API_SERVICE = "Checkbot_api" NOT_CONFIGURED = "nicht konfiguriert" +NOT_YET_RUN = "noch nicht gelaufen" +WARN_DETAILS = (NOT_CONFIGURED, NOT_YET_RUN) MAX_TEXT = 200 OK, WARN, CRIT = 0, 1, 2 @@ -45,8 +47,8 @@ def evaluate(name: str, result: dict | None) -> str: detail = result.get("detail", "") if result.get("ok"): return line(OK, service, detail or "ok") - if detail == NOT_CONFIGURED: - return line(WARN, service, NOT_CONFIGURED) + if detail in WARN_DETAILS: + return line(WARN, service, detail) return line(CRIT, service, detail or "Fehler") diff --git a/tests/test_checkmk_fallbacks.py b/tests/test_checkmk_fallbacks.py new file mode 100644 index 0000000..45cc663 --- /dev/null +++ b/tests/test_checkmk_fallbacks.py @@ -0,0 +1,120 @@ +import json + +import httpx + +from app.checkmk import (DOWNTIME_MODE_QUERY, HOST_QUERY_GET, HOST_QUERY_POST, CheckmkClient, CheckmkError, + Downtime, HostState, ServiceState, downtimes_from_states, status_hint) + +def make_client(handler): + return CheckmkClient("https://cmk.local", "prod", "bot", "s3cret", transport=httpx.MockTransport(handler)) + + +def host_row(name="srv1", state=1): + return {"extensions": {"name": name, "state": state, "plugin_output": "x", "last_state_change": 1, + "scheduled_downtime_depth": 0, "acknowledged": 0}} + + +async def test_problem_hosts_with_fallback_switches_to_get_and_remembers_mode(): + calls = [] + + def handler(request): + calls.append(request.method) + if request.method == "POST": + return httpx.Response(422, text="Spalten abgelehnt") + return httpx.Response(200, json={"value": [host_row(state=0), host_row("srv2", state=2)]}) + + client = make_client(handler) + hosts, mode = await client.problem_hosts_with_fallback() + assert mode == HOST_QUERY_GET + assert [h.host for h in hosts] == ["srv2"] + assert calls == ["POST", "GET"] + + calls.clear() + await client.problem_hosts() + assert calls == ["GET"] + + +async def test_problem_hosts_with_fallback_raises_when_both_fail(): + def handler(request): + return httpx.Response(404, text="weg") + + try: + await make_client(handler).problem_hosts_with_fallback() + except CheckmkError as exc: + assert exc.brief == "HTTP 404" + else: + raise AssertionError("expected CheckmkError") + + +async def test_selftest_hosts_reports_raw_rows_and_mode(): + def handler(request): + return httpx.Response(200, json={"value": [host_row(state=2)]}) + + rows, hosts, mode = await make_client(handler).selftest_hosts() + assert mode == HOST_QUERY_POST + assert rows[0]["name"] == "srv1" + assert [h.state for h in hosts] == [3] + + +async def test_selftest_services_keeps_raw_rows_when_parse_fails(): + body = {"value": [{"extensions": {"description": "s", "state": 2}}]} # host_name fehlt (Pflichtspalte) + + rows, services = await make_client(lambda r: httpx.Response(200, json=body)).selftest_services() + assert rows == [{"description": "s", "state": 2}] + assert services == [] + + +async def test_downtimes_with_fallback_uses_query_when_type_rejected(): + seen = [] + + def handler(request): + seen.append(request) + if "downtime_type" in request.url.params: + return httpx.Response(422, text="abgelehnt") + return httpx.Response(200, json={"value": [{"extensions": { + "host_name": "web01", "service_description": "", "start_time": "", "end_time": "", + "comment": "", "author": ""}}]}) + + downtimes, mode = await make_client(handler).downtimes_with_fallback() + assert mode == DOWNTIME_MODE_QUERY + assert len(downtimes) == 2 + queries = [json.loads(r.url.params["query"]) for r in seen if "query" in r.url.params] + assert {q["right"] for q in queries} == {"0", "1"} + + +async def test_downtimes_with_fallback_propagates_when_both_fail(): + def handler(request): + return httpx.Response(404, text="weg") + + try: + await make_client(handler).downtimes_with_fallback() + except CheckmkError as exc: + assert exc.brief == "HTTP 404" + else: + raise AssertionError("expected CheckmkError") + + +def test_downtimes_from_states_derives_from_scheduled_downtime_depth(): + services = [ServiceState("h1", "s1", 2, True, "", 1, True, False), ServiceState("h1", "s2", 0, True, "", 1, False, False)] + hosts = [HostState("h2", 2, "", 1, True, False)] + derived = downtimes_from_states(services, hosts) + assert derived == [Downtime("h2", None, "", "", "", ""), Downtime("h1", "s1", "", "", "", "")] + + +def test_status_hint_mapping_never_touches_brief(): + assert status_hint(401) == "Zugangsdaten prüfen" + assert status_hint(403) == "Rechte des Automation-Users prüfen" + assert status_hint(404) == "Pfad oder Site prüfen" + assert status_hint(422) == "Anfrage von Checkmk abgelehnt (Format/Spalten prüfen)" + assert status_hint(503) == "Checkmk-Serverfehler" + assert status_hint(418) == "" + + +async def test_request_error_carries_hint_but_brief_is_unchanged(): + client = make_client(lambda r: httpx.Response(401, text="geheim")) + try: + await client.problem_services() + except CheckmkError as exc: + assert exc.brief == "HTTP 401" and exc.hint == "Zugangsdaten prüfen" + else: + raise AssertionError("expected CheckmkError") diff --git a/tests/test_checkmk_reads.py b/tests/test_checkmk_reads.py index 9de7142..c8444af 100644 --- a/tests/test_checkmk_reads.py +++ b/tests/test_checkmk_reads.py @@ -55,6 +55,31 @@ async def test_problems_alias(): assert len(await client.problems()) == 1 +async def test_service_status_accepts_members_instead_of_value(): + body = {"members": [item()]} + result = await make_client(lambda r: httpx.Response(200, json=body)).service_status("web01", "CPU load") + assert result.service == "CPU load" and result.state == 2 + + +async def test_state_type_as_string_hard_and_soft(): + def hard_soft(state_type_value): + extensions = item()["extensions"] + extensions["state_type"] = state_type_value + return {"extensions": extensions} + + client = make_client(lambda r: httpx.Response(200, json={"value": [hard_soft("hard")]})) + (service,) = await client.problem_services() + assert service.hard is True + + client = make_client(lambda r: httpx.Response(200, json={"value": [hard_soft("SOFT")]})) + (service,) = await client.problem_services() + assert service.hard is False + + client = make_client(lambda r: httpx.Response(200, json={"value": [hard_soft(None)]})) + (service,) = await client.problem_services() + assert service.hard is False + + async def test_downtimes_all_and_by_host(): seen = [] diff --git a/tests/test_health_selftest.py b/tests/test_health_selftest.py new file mode 100644 index 0000000..5e547bc --- /dev/null +++ b/tests/test_health_selftest.py @@ -0,0 +1,71 @@ +import os + +from app.constants import CHECKMK_SECRET, CHECKMK_SITE, CHECKMK_URL, CHECKMK_USER +from app.crypto import SecretBox +from app.db import Database +from app.health import build_checks, run_checks +from app.selftest import (CHECK_VERSION, STATUS_DEVIATED, STATUS_FAILED, STATUS_OK, CheckResult, + SelftestRepository, SelftestRun) +from app.settings import SettingsStore + + +def make_store(): + db = Database(":memory:") + return db, SettingsStore(db, SecretBox(os.urandom(32))) + + +def configure(store): + store.set(CHECKMK_URL, "https://cmk.local", "t") + store.set(CHECKMK_SITE, "prod", "t") + store.set(CHECKMK_USER, "bot", "t") + store.set(CHECKMK_SECRET, "s3cret", "t", secret=True) + + +def only_checkmk_api(store, selftest_repo=None) -> dict: + # Nur den einen Teilcheck ausführen, damit "checkmk" nicht mit (hier nicht gemocktem) echtem Netzwerkzugriff läuft. + checks = build_checks(store, selftest_repo=selftest_repo) + return {"checkmk_api": checks["checkmk_api"]} + + +async def test_checkmk_api_missing_repo_is_not_configured(): + _, store = make_store() + configure(store) + result = await run_checks(only_checkmk_api(store)) + assert result["checks"]["checkmk_api"] == {"ok": False, "detail": "nicht konfiguriert"} + + +async def test_checkmk_api_without_checkmk_config_is_not_configured(): + db, store = make_store() + result = await run_checks(only_checkmk_api(store, SelftestRepository(db))) + assert result["checks"]["checkmk_api"] == {"ok": False, "detail": "nicht konfiguriert"} + + +async def test_checkmk_api_not_yet_run_behaves_like_not_configured(): + db, store = make_store() + configure(store) + result = await run_checks(only_checkmk_api(store, SelftestRepository(db))) + assert result["checks"]["checkmk_api"] == {"ok": False, "detail": "noch nicht gelaufen"} + assert result["ok"] is False + + +async def test_checkmk_api_ok(): + db, store = make_store() + configure(store) + repo = SelftestRepository(db) + repo.save_run(SelftestRun(1, "2.5.0", [CheckResult(CHECK_VERSION, STATUS_OK, "ok")])) + result = await run_checks(only_checkmk_api(store, repo)) + check = result["checks"]["checkmk_api"] + assert check["ok"] is True and "2.5.0" in check["detail"] + + +async def test_checkmk_api_deviated_and_failed_are_not_ok(): + db, store = make_store() + configure(store) + repo = SelftestRepository(db) + repo.save_run(SelftestRun(1, "2.5.0", [CheckResult(CHECK_VERSION, STATUS_DEVIATED, "Spalte fehlt")])) + result = await run_checks(only_checkmk_api(store, repo)) + assert result["checks"]["checkmk_api"]["ok"] is False and result["ok"] is False + + repo.save_run(SelftestRun(2, "2.5.0", [CheckResult(CHECK_VERSION, STATUS_FAILED, "kaputt")])) + result = await run_checks(only_checkmk_api(store, repo)) + assert result["checks"]["checkmk_api"]["ok"] is False diff --git a/tests/test_selftest.py b/tests/test_selftest.py new file mode 100644 index 0000000..4fe1fae --- /dev/null +++ b/tests/test_selftest.py @@ -0,0 +1,179 @@ +import os + +import httpx + +from app.checkmk import CheckmkClient +from app.crypto import SecretBox +from app.db import Database +from app.selftest import (CHECK_DOWNTIMES, CHECK_HOSTS, CHECK_SERVICES, CHECK_VERSION, STATUS_DEVIATED, + STATUS_FAILED, STATUS_OK, SelftestNotConfigured, SelftestRepository, SelftestRun, + SelftestService, message_for) +from app.selftest.checks import check_downtimes, check_hosts, check_services, check_version +from app.selftest.model import CheckResult +from app.settings import SettingsStore + + +def make_client(handler): + return CheckmkClient("https://cmk.local", "prod", "bot", "s3cret", transport=httpx.MockTransport(handler)) + + +def service_row(host="h1", service="s1", state=2, **extra): + data = {"host_name": host, "description": service, "state": state, "state_type": 1, "plugin_output": "x", + "last_state_change": 1, "scheduled_downtime_depth": 0, "is_flapping": 0, "acknowledged": 0} + data.update(extra) + return {"extensions": data} + + +def host_row(name="h2", state=1, **extra): + data = {"name": name, "state": state, "plugin_output": "x", "last_state_change": 1, + "scheduled_downtime_depth": 0, "acknowledged": 0} + data.update(extra) + return {"extensions": data} + + +def good_handler(request): + if request.url.path.endswith("/version"): + return httpx.Response(200, json={"versions": {"checkmk": "2.5.0p1"}}) + if request.url.path.endswith("/service/collections/all"): + return httpx.Response(200, json={"value": [service_row()]}) + if request.url.path.endswith("/host/collections/all"): + return httpx.Response(200, json={"value": [host_row()]}) + if request.url.path.endswith("/downtime/collections/all"): + return httpx.Response(200, json={"value": []}) + raise AssertionError(request.url.path) + + +async def test_check_version_ok_deviated_and_failed(): + ok, version = await check_version(make_client(good_handler)) + assert ok.status == STATUS_OK and ok.name == CHECK_VERSION and version == "2.5.0p1" + + deviated, version = await check_version(make_client(lambda r: httpx.Response(200, json={"versions": {}}))) + assert deviated.status == STATUS_DEVIATED and version == "" + + failed, version = await check_version(make_client(lambda r: httpx.Response(401, text="x"))) + assert failed.status == STATUS_FAILED and version == "" + + +async def test_check_services_reports_missing_columns(): + client = make_client(lambda r: httpx.Response(200, json={"value": [service_row()]})) + result, services = await check_services(client) + assert result.status == STATUS_OK and len(services) == 1 + + missing_optional = make_client(lambda r: httpx.Response( + 200, json={"value": [{"extensions": {"host_name": "h", "description": "s", "state": 2}}]})) + result, services = await check_services(missing_optional) + assert result.status == STATUS_DEVIATED and "last_state_change" in result.text + + missing_required = make_client(lambda r: httpx.Response( + 200, json={"value": [{"extensions": {"description": "s", "state": 2}}]})) + result, services = await check_services(missing_required) + assert result.status == STATUS_FAILED and "host_name" in result.text + + +async def test_check_hosts_reports_get_fallback_as_deviated(): + def handler(request): + if request.method == "POST": + return httpx.Response(422, text="x") + return httpx.Response(200, json={"value": [host_row(state=2)]}) + + result, hosts = await check_hosts(make_client(handler)) + assert result.status == STATUS_DEVIATED and "GET-Fallback" in result.text and len(hosts) == 1 + + +async def test_check_downtimes_falls_back_to_query_then_to_derived(): + def query_fallback(request): + if "downtime_type" in request.url.params: + return httpx.Response(422, text="x") + return httpx.Response(200, json={"value": []}) + + result = await check_downtimes(make_client(query_fallback), [], []) + assert result.status == STATUS_DEVIATED and "query" in result.text + + def all_fail(request): + return httpx.Response(404, text="x") + + from app.checkmk import ServiceState + services = [ServiceState("h1", "s1", 2, True, "", 1, True, False)] + result = await check_downtimes(make_client(all_fail), services, []) + assert result.status == STATUS_DEVIATED and "abgeleitet" in result.text and "1 Downtime" in result.text + + +def make_env(): + db = Database(":memory:") + store = SettingsStore(db, SecretBox(os.urandom(32))) + return db, store + + +def configure(store): + from app.constants import CHECKMK_SECRET, CHECKMK_SITE, CHECKMK_URL, CHECKMK_USER + store.set(CHECKMK_URL, "https://cmk.local", "t") + store.set(CHECKMK_SITE, "prod", "t") + store.set(CHECKMK_USER, "bot", "t") + store.set(CHECKMK_SECRET, "s3cret", "t", secret=True) + + +async def test_service_run_not_configured(): + db, store = make_env() + service = SelftestService(store, SelftestRepository(db)) + try: + await service.run() + except SelftestNotConfigured: + pass + else: + raise AssertionError("expected SelftestNotConfigured") + + +async def test_service_run_persists_and_detects_version_change(monkeypatch): + db, store = make_env() + configure(store) + repo = SelftestRepository(db) + service = SelftestService(store, repo, + client_factory=lambda s: make_client(good_handler), clock=lambda: 1_800_000_000.0) + run = await service.run() + assert run.version == "2.5.0p1" and run.version_changed is False + assert repo.last_run().version == "2.5.0p1" + + def newer(request): + if request.url.path.endswith("/version"): + return httpx.Response(200, json={"versions": {"checkmk": "2.5.0p2"}}) + return good_handler(request) + + service2 = SelftestService(store, repo, client_factory=lambda s: make_client(newer), + clock=lambda: 1_800_000_100.0) + run2 = await service2.run() + assert run2.version == "2.5.0p2" and run2.version_changed is True + + +def test_overall_status_priority(): + ok = CheckResult("a", STATUS_OK, "") + deviated = CheckResult("b", STATUS_DEVIATED, "") + failed = CheckResult("c", STATUS_FAILED, "") + assert SelftestRun(1, "v", [ok]).overall_status == STATUS_OK + assert SelftestRun(1, "v", [ok, deviated]).overall_status == STATUS_DEVIATED + assert SelftestRun(1, "v", [ok, deviated, failed]).overall_status == STATUS_FAILED + + +def run_with(status): + return SelftestRun(1, "2.5.0", [CheckResult(CHECK_VERSION, status, "x")]) + + +def test_message_for_no_previous_is_silent(): + assert message_for(None, run_with(STATUS_OK)) is None + + +def test_message_for_same_status_is_silent(): + assert message_for(run_with(STATUS_OK), run_with(STATUS_OK)) is None + + +def test_message_for_status_change_to_failed_and_back_to_ok(): + text = message_for(run_with(STATUS_OK), run_with(STATUS_FAILED)) + assert text is not None and "fehler" in text + text = message_for(run_with(STATUS_FAILED), run_with(STATUS_OK)) + assert text == "Selbsttest wieder ok" + + +def test_message_for_version_change(): + previous = SelftestRun(1, "2.4.0", [CheckResult(CHECK_VERSION, STATUS_OK, "x")]) + current = SelftestRun(2, "2.5.0", [CheckResult(CHECK_VERSION, STATUS_OK, "x")], version_changed=True) + text = message_for(previous, current) + assert text == "Checkmk-Version von 2.4.0 auf 2.5.0, Selbsttest: ok" diff --git a/tests/test_selftest_runner.py b/tests/test_selftest_runner.py new file mode 100644 index 0000000..06ea6ec --- /dev/null +++ b/tests/test_selftest_runner.py @@ -0,0 +1,142 @@ +from datetime import datetime + +import pytest + +from app.db import Database +from app.selftest import CHECK_VERSION, STATUS_FAILED, STATUS_OK, CheckResult, SelftestRun, should_run +from app.selftest.runner import SelftestRunner +from app.selftest.service import SelftestNotConfigured + + +def at(day, hour, minute=0): + return datetime(2026, 11, day, hour, minute) + + +def stamp(moment): + return moment.timestamp() + + +@pytest.mark.parametrize("now, last, startup, expected", [ + (at(6, 3, 59), None, False, False), + (at(6, 4, 0), None, False, True), + (at(6, 3, 59), None, True, True), + (at(6, 10), stamp(at(6, 4, 5)), True, False), + (at(6, 10), stamp(at(6, 4, 5)), False, False), + (at(6, 1), stamp(at(5, 4, 5)), True, True), + (at(6, 1), stamp(at(5, 4, 5)), False, False), +]) +def test_should_run(now, last, startup, expected): + assert should_run(now, last, startup) is expected + + +class Moment: + def __init__(self, value): + self.value = value + + def __call__(self): + return self.value + + +class FakeNotifier: + def __init__(self): + self.calls = [] + + async def selftest(self, ok, text): + self.calls.append((ok, text)) + + +class FakeService: + def __init__(self, runs=None, error=None): + self.runs = list(runs or []) + self.error = error + self.record = None + self.call_count = 0 + + def last_run(self): + return self.record + + async def run(self): + self.call_count += 1 + if self.error: + raise self.error + run = self.runs.pop(0) + self.record = run + return run + + +def run_with(ts, status, version_changed=False): + return SelftestRun(ts, "2.5.0", [CheckResult(CHECK_VERSION, status, "x")], version_changed) + + +def make_runner(service, notify_enabled=lambda: True, enabled=lambda: True, clock=None): + db = Database(":memory:") + return db, SelftestRunner(db, service, FakeNotifier(), enabled, notify_enabled, + clock=clock or Moment(at(6, 10))) + + +async def test_tick_runs_once_per_day_and_respects_enabled(): + service = FakeService([run_with(int(at(6, 10).timestamp()), STATUS_OK)]) + clock = Moment(at(6, 10)) + db, runner = make_runner(service, clock=clock) + assert await runner.tick(startup=True) is True + clock.value = at(6, 11) + assert await runner.tick(startup=False) is False + assert service.call_count == 1 + + +async def test_tick_disabled_never_runs(): + service = FakeService([run_with(1, STATUS_OK)]) + db, runner = make_runner(service, enabled=lambda: False) + assert await runner.tick(startup=True) is False + assert service.call_count == 0 + + +async def test_tick_not_configured_is_tolerated(): + service = FakeService(error=SelftestNotConfigured()) + db, runner = make_runner(service) + assert await runner.tick(startup=True) is True + + +async def test_manual_rate_limit(): + service = FakeService([run_with(1, STATUS_OK), run_with(2, STATUS_OK)]) + db, runner = make_runner(service) + assert runner.try_manual() is True + assert runner.try_manual() is False + + +async def test_run_manual_returns_none_when_not_configured(): + service = FakeService(error=SelftestNotConfigured()) + db, runner = make_runner(service) + assert await runner.run_manual("admin1") is None + + +async def test_run_manual_writes_audit_entry(): + service = FakeService([run_with(1, STATUS_OK)]) + db, runner = make_runner(service) + await runner.run_manual("admin1") + rows = db.query("SELECT user, action, target FROM audit WHERE action='selftest.run'") + assert len(rows) == 1 and rows[0]["user"] == "admin1" and rows[0]["target"] == "selftest" + + +async def test_notify_fires_on_status_change_but_not_on_repeat(): + service = FakeService([run_with(1, STATUS_OK), run_with(2, STATUS_FAILED), run_with(3, STATUS_FAILED)]) + db = Database(":memory:") + notifier = FakeNotifier() + runner = SelftestRunner(db, service, notifier, lambda: True, lambda: True, clock=Moment(at(6, 10))) + + await runner.run_manual("admin1") + assert notifier.calls == [] # erster Lauf, kein Vorlauf zum Vergleich + + await runner.run_manual("admin1") + assert len(notifier.calls) == 1 and notifier.calls[0][0] is False + + await runner.run_manual("admin1") + assert len(notifier.calls) == 1 # gleicher Status wie zuvor, keine Wiederholung + + +async def test_notify_disabled_sends_nothing(): + service = FakeService([run_with(1, STATUS_OK), run_with(2, STATUS_FAILED)]) + db, runner = make_runner(service, notify_enabled=lambda: False) + await runner.run_manual("admin1") + await runner.run_manual("admin1") + assert runner._notifier.calls == [] diff --git a/tests/test_web_selftest.py b/tests/test_web_selftest.py new file mode 100644 index 0000000..57cf96a --- /dev/null +++ b/tests/test_web_selftest.py @@ -0,0 +1,78 @@ +from app.selftest import CHECK_VERSION, STATUS_OK, CheckResult, SelftestRun +from tests.test_web import VIEWER, admin, app, client, login_ok, post, users, viewer # noqa: F401 + +PAGE = "/selftest" +RUN_PATH = "/selftest/run" + + +def test_page_requires_login(client): + assert client.get(PAGE).status_code == 303 + + +def test_viewer_sees_page_without_run_button(client, viewer): + login_ok(client, VIEWER) + page = client.get(PAGE) + assert page.status_code == 200 and "Noch kein Lauf" in page.text + assert 'action="/selftest/run"' not in page.text + + +def test_admin_sees_run_button(client, admin): + login_ok(client) + page = client.get(PAGE) + assert page.status_code == 200 and 'action="/selftest/run"' in page.text + + +def test_viewer_cannot_trigger_run(client, viewer): + login_ok(client, VIEWER) + assert post(client, RUN_PATH, PAGE).status_code == 403 + + +def test_run_requires_csrf(client, admin): + login_ok(client) + assert client.post(RUN_PATH).status_code == 403 + + +def test_manual_run_not_configured_flashes_error(client, admin): + login_ok(client) + response = post(client, RUN_PATH, PAGE) + assert response.status_code == 303 + assert "nicht konfiguriert" in client.get(PAGE).text + + +def test_manual_run_shows_result_and_writes_audit(client, app, admin, monkeypatch): + login_ok(client) + run = SelftestRun(1_800_000_000, "2.5.0", [CheckResult(CHECK_VERSION, STATUS_OK, "Checkmk 2.5.0")]) + + async def fake_run(self): + return run + + service = app.state.runtime.selftest + monkeypatch.setattr(type(service), "run", fake_run) + post(client, RUN_PATH, PAGE) + page = client.get(PAGE) + assert "Checkmk 2.5.0" in page.text and "2.5.0" in page.text + rows = app.state.db.query("SELECT user, action, target FROM audit WHERE action='selftest.run'") + assert len(rows) == 1 and rows[0]["user"] == "admin1" and rows[0]["target"] == "selftest" + + +def test_manual_run_is_rate_limited(client, admin, monkeypatch): + login_ok(client) + post(client, RUN_PATH, PAGE) + response = post(client, RUN_PATH, PAGE) + assert "warten" in client.get(PAGE).text + + +def test_settings_section_toggles_require_admin(client, viewer): + login_ok(client, VIEWER) + assert client.get("/settings/selftest").status_code == 403 + + +def test_settings_section_save_updates_flags(client, app, admin): + login_ok(client) + page = client.get("/settings/selftest") + assert page.status_code == 200 + response = post(client, "/settings/selftest", "/settings/selftest", + selftest_enabled="", selftest_notify="1") + assert response.status_code == 303 + store = app.state.store + assert store.get("selftest.enabled") == "0" and store.get("selftest.notify") == "1"