From f34fc6d686992c51b5e621c1dd0e86ba702eaa4f Mon Sep 17 00:00:00 2001 From: patrick Date: Thu, 12 Nov 2026 23:38:07 +0100 Subject: [PATCH] =?UTF-8?q?feat:=20Ausgangslog=20f=C3=BCr=20Messenger-Nach?= =?UTF-8?q?richten?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LoggingAdapter protokolliert send/edit (Matrix zusätzlich reply) inkl. Fehlversuche - Tabelle outbound_log (Migration 10), Aufbewahrung retention.outbound_days (30 Tage) - Seite /messages und CLI checkbot messages mit Filtern - Text gekappt (4000) und Secrets maskiert Co-Authored-By: Claude Sonnet 5.5 --- CLAUDE.md | 9 ++ app/chat/__init__.py | 4 +- app/chat/outbound.py | 81 ++++++++++++++++++ app/chat/registry.py | 26 ++++-- app/chatops.py | 21 ++++- app/cli.py | 28 ++++++ app/constants.py | 2 + app/main.py | 5 +- app/migrations.py | 16 ++++ app/outbound/__init__.py | 4 + app/outbound/repository.py | 75 ++++++++++++++++ app/outbound/since.py | 14 +++ app/retention/policy.py | 12 ++- app/retention/repository.py | 5 +- app/runtime.py | 9 +- app/web/messages_pages.py | 34 ++++++++ app/web/retention_section.py | 1 + app/web/templates/base.html | 1 + app/web/templates/messages.html | 48 +++++++++++ deploy/checkbot-cli | 2 +- tests/test_cli_messages.py | 42 +++++++++ tests/test_outbound_logging_adapter.py | 113 +++++++++++++++++++++++++ tests/test_outbound_repository.py | 73 ++++++++++++++++ tests/test_retention.py | 30 ++++--- tests/test_web_messages.py | 53 ++++++++++++ tests/test_web_retention.py | 11 ++- 26 files changed, 684 insertions(+), 35 deletions(-) create mode 100644 app/chat/outbound.py create mode 100644 app/outbound/__init__.py create mode 100644 app/outbound/repository.py create mode 100644 app/outbound/since.py create mode 100644 app/web/messages_pages.py create mode 100644 app/web/templates/messages.html create mode 100644 tests/test_cli_messages.py create mode 100644 tests/test_outbound_logging_adapter.py create mode 100644 tests/test_outbound_repository.py create mode 100644 tests/test_web_messages.py diff --git a/CLAUDE.md b/CLAUDE.md index e3647f9..85aa7cc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -105,6 +105,15 @@ tests/ - `deploy/checkmk/checkbot-local-check`: neuer Service `Checkbot_checkmk_api`, Detailtext `noch nicht gelaufen` wird wie `nicht konfiguriert` als WARN gewertet (`WARN_DETAILS`). - Annahmen/Abweichungen: "sofort bei Versionswechsel" löst keinen zusätzlichen Sofort-Lauf aus, sondern markiert nur den laufenden Lauf als Versionswechsel für die Chat-Meldung (im Auftrag so vorgesehen: "sofort erneut laufen lassen bzw. den aktuellen Lauf kennzeichnen"). Fehlt der Parser-Vergleich an Spalten, weil aktuell keine Problem-Services/-Hosts vorliegen (leere Ergebnisliste), kann der jeweilige Teilcheck die Spalten nicht prüfen und meldet `ok`, auch wenn eine Spalte in Wahrheit fehlen würde – eine Grenze des Ansatzes, der nur die echten Problem-Listen liest statt eines eigenen Schema-Endpunkts. `.hint` an `CheckmkError` ist zusätzlich zu `.brief`, nicht statt dessen, damit bestehende Tests mit exaktem `brief`-Text (z. B. `"HTTP 401"`) unverändert bleiben. +## Ausgangslog (ungetestet) +- Migration Index 10 (elfter Eintrag in `app/migrations.py`): Tabelle `outbound_log` (ts, backend, target, action `send`/`edit`, message_id, silent, ok, error, text) plus Indizes auf `ts` und `(backend, ts)`. +- `app/chat/outbound.py`: Decorator `LoggingAdapter` um einen `ChatAdapter`, protokolliert `send`/`edit` (und bei Matrix zusätzlich `reply`, weil Befehlsantworten dort nicht über `send` laufen) über ein `OutboundSink`; alle anderen Attribute/Methoden (`name`, `check`, `aclose`, …) reicht `__getattr__` durch. Sendefehler werden protokolliert und unverändert weitergeworfen, ein Fehler beim Protokollieren selbst wird nur geloggt (`log.warning`, Typname) und stört das Senden nicht. Text wird auf 4000 Zeichen gekappt, Fehlertext auf 200; beide werden mit den konfigurierten Secrets (`mask_secret`) und den Mustern aus `app/logsetup.py` (`mask_patterns`) maskiert. +- Eingehängt in `app/chat/registry.py`: `build_telegram`/`build_matrix` (und darüber `build_adapters`) nehmen optional `sink`/`secrets` und wrappen den fertigen Adapter. Zwei Aufrufstellen reichen den Sink durch: `Runtime.registry` (`app/runtime.py`, Factory mit `sink=self.outbound`) für den Deliverer und `SystemNotifier`, sowie `ChatOps` (`app/chatops.py`) für `TelegramListener`/`MatrixListener` (Befehlsantworten, Callbacks). Verbindungstests in `app/web/telegram_section.py`/`matrix_section.py` bauen ihre Adapter bewusst direkt (ohne `build_telegram`/`build_matrix`) und bleiben daher unprotokolliert – das sind Admin-Testpings, keine Betriebsnachrichten. +- Setting `retention.outbound_days` (Default 30, 1–3650) in der bestehenden Section "Aufbewahrung", täglicher Lauf löscht `outbound_log` wie die anderen Tabellen in Batches zu 1000. +- Seite `/messages` (`app/web/messages_pages.py` + `messages.html`): Filter Backend, Textsuche (auf Text und Ziel, `%`/`_` escaped, parametrisiert), nur Fehler, Zeitraum (`2h`/`1d`, …), Paging. Lesbar für alle angemeldeten Rollen, kein Admin-Sonderrecht nötig (nur Lesen). Text wird über Jinja-Autoescape angezeigt (kein `|safe`), aufklappbar per `
`. +- CLI `checkbot messages [--grep TEXT] [--backend B] [--errors] [--since 2h|1d] [--limit N]` (`app/cli.py`, `app/outbound/since.py:parse_since`), öffnet die DB wie die anderen CLI-Befehle (kein eigener Read-only-Modus vorhanden). `deploy/checkbot-cli` Usage-Text ergänzt. +- Annahmen/Abweichungen: Gekappt wird der nach Datenschutzmodus (voll/kurz) schon zusammengesetzte Text, also genau das, was der Adapter bekommt. Verbindungstests (s.o.) sind bewusst ausgenommen. `OutboundRepository`/CLI öffnen die Datenbank ohne eigenen Read-only-Modus, weil es dafür noch kein Vorbild im Code gibt. + ## Weiterer Weg (entschieden) Reihenfolge: Testbasis, erster Lauf im Test-LXC (`pytest`), Korrekturrunde, Phase 6, Phase 6b, Phase 6c, Phase 8, Abnahme, Parallelbetrieb, Tag `v0.1.0`. Verfeinerungen passieren am Live-System, nicht vorab. - Testbasis: `pytest` manuell im Test-LXC (Dev-venv mit `requirements-dev.txt`), dazu `.gitea/workflows/test.yml` für einen späteren Gitea-Actions-Runner. Logging als Zeilenlog nach journald, Level über `CHECKBOT_LOG_LEVEL` (Default INFO), nie Secrets in Logs (Test für die Adapter). Zeitzone über `CHECKBOT_TZ` im Setup-Skript, Default `Europe/Berlin`. diff --git a/app/chat/__init__.py b/app/chat/__init__.py index 3ffe4fc..2a91c82 100644 --- a/app/chat/__init__.py +++ b/app/chat/__init__.py @@ -1,12 +1,14 @@ from .base import Button, Buttons, ChatAdapter, ChatError, MessageRef, Target, json_body, mask_secret from . import callback_data from .matrix import BACKEND_MATRIX, MatrixAdapter, MatrixError +from .outbound import LoggingAdapter from .registry import AdapterRegistry, build_adapters, build_matrix, build_telegram, default_targets from .targets import MATRIX_ROOM_ID_RE, MATRIX_USER_ID_RE, TELEGRAM_CHAT_ID_RE, valid_target from .telegram import BACKEND_TELEGRAM, MESSAGE_MAX as TELEGRAM_MESSAGE_MAX, TelegramAdapter, TelegramError, truncate_html __all__ = [ - "AdapterRegistry", "BACKEND_MATRIX", "Button", "Buttons", "callback_data", "BACKEND_TELEGRAM", "ChatAdapter", "ChatError", "MATRIX_ROOM_ID_RE", + "AdapterRegistry", "BACKEND_MATRIX", "Button", "Buttons", "callback_data", "BACKEND_TELEGRAM", "ChatAdapter", + "ChatError", "LoggingAdapter", "MATRIX_ROOM_ID_RE", "MATRIX_USER_ID_RE", "MatrixAdapter", "MatrixError", "MessageRef", "TELEGRAM_CHAT_ID_RE", "Target", "TELEGRAM_MESSAGE_MAX", "TelegramAdapter", "TelegramError", "build_adapters", "build_matrix", "build_telegram", "default_targets", "json_body", "mask_secret", "truncate_html", "valid_target", diff --git a/app/chat/outbound.py b/app/chat/outbound.py new file mode 100644 index 0000000..3c513c3 --- /dev/null +++ b/app/chat/outbound.py @@ -0,0 +1,81 @@ +import logging +import time +from typing import Callable, Protocol + +from ..logsetup import mask_patterns +from .base import Buttons, ChatAdapter, MessageRef, mask_secret + +log = logging.getLogger(__name__) + +TEXT_MAX = 4000 +ERROR_MAX = 200 +ACTION_SEND = "send" +ACTION_EDIT = "edit" + + +class OutboundSink(Protocol): + def record(self, ts: int, backend: str, target: str, action: str, message_id: str, silent: bool, ok: bool, + error: str | None, text: str) -> None: ... + + +def _mask(text: str, secrets: list[str]) -> str: + for secret in secrets: + text = mask_secret(text, secret) + return mask_patterns(text) + + +class LoggingAdapter: + """Protokolliert send/edit (und, falls vorhanden, reply) eines ChatAdapter; Sendefehler laufen unverändert durch.""" + + def __init__(self, inner: ChatAdapter, sink: OutboundSink, secrets: Callable[[], list[str]] = lambda: (), + clock: Callable[[], float] = time.time): + self._inner = inner + self._sink = sink + self._secrets = secrets + self._clock = clock + + def __getattr__(self, name): + return getattr(self._inner, name) + + def _log(self, backend: str, target: str, action: str, message_id: str, silent: bool, ok: bool, + error: str | None, text: str) -> None: + try: + masked = _mask(text, list(self._secrets()))[:TEXT_MAX] + masked_error = _mask(error, list(self._secrets()))[:ERROR_MAX] if error else None + self._sink.record(int(self._clock()), backend, target, action, message_id, silent, ok, masked_error, + masked) + except Exception as exc: + log.warning("Protokollieren der Ausgangsnachricht fehlgeschlagen: %s", type(exc).__name__) + + async def send(self, target: str, text: str, buttons: Buttons = (), silent: bool = False, + **kwargs) -> MessageRef: + try: + ref = await self._inner.send(target, text, buttons=buttons, silent=silent, **kwargs) + except Exception as exc: + self._log(self._inner.name, target, ACTION_SEND, "", silent, False, _error_text(exc), text) + raise + self._log(self._inner.name, target, ACTION_SEND, ref.message_id, silent, True, None, text) + return ref + + async def edit(self, ref: MessageRef, text: str, buttons: Buttons = (), silent: bool = False, + **kwargs) -> None: + try: + await self._inner.edit(ref, text, buttons=buttons, silent=silent, **kwargs) + except Exception as exc: + self._log(ref.backend, ref.target, ACTION_EDIT, ref.message_id, silent, False, _error_text(exc), text) + raise + self._log(ref.backend, ref.target, ACTION_EDIT, ref.message_id, silent, True, None, text) + + async def reply(self, target: str, text: str, html: str | None, event_id: str): + # Matrix-Befehlsantworten laufen über reply(), nicht über send(); zählt für das Ausgangslog als 'send'. + try: + new_event_id = await self._inner.reply(target, text, html, event_id) + except Exception as exc: + self._log(self._inner.name, target, ACTION_SEND, "", False, False, _error_text(exc), text) + raise + self._log(self._inner.name, target, ACTION_SEND, new_event_id, False, True, None, text) + return new_event_id + + +def _error_text(exc: Exception) -> str: + return f"{type(exc).__name__}: {exc}" diff --git a/app/chat/registry.py b/app/chat/registry.py index 947d22d..a727b59 100644 --- a/app/chat/registry.py +++ b/app/chat/registry.py @@ -5,20 +5,31 @@ from ..constants import (ADAPTER_SETTING_KEYS, DEFAULT_TELEGRAM_API_BASE, FLAG_O TELEGRAM_CHAT_ID, TELEGRAM_ENABLED, TELEGRAM_TOKEN) from .base import ChatAdapter, Target from .matrix import BACKEND_MATRIX, MatrixAdapter +from .outbound import LoggingAdapter, OutboundSink from .telegram import BACKEND_TELEGRAM, TelegramAdapter -def build_telegram(store, timeout: float | None = None) -> TelegramAdapter | None: +def _wrap(adapter: ChatAdapter | None, sink: OutboundSink | None, + secrets: Callable[[], list[str]]) -> ChatAdapter | None: + if adapter is None or sink is None: + return adapter + return LoggingAdapter(adapter, sink, secrets) + + +def build_telegram(store, timeout: float | None = None, sink: OutboundSink | None = None, + secrets: Callable[[], list[str]] = lambda: ()) -> TelegramAdapter | None: if store.get(TELEGRAM_ENABLED) != FLAG_ON: return None token, chat_id = store.get(TELEGRAM_TOKEN), store.get(TELEGRAM_CHAT_ID) if not (token and chat_id): return None options = {} if timeout is None else {"timeout": timeout} - return TelegramAdapter(token, chat_id, store.get(TELEGRAM_API_BASE) or DEFAULT_TELEGRAM_API_BASE, **options) + adapter = TelegramAdapter(token, chat_id, store.get(TELEGRAM_API_BASE) or DEFAULT_TELEGRAM_API_BASE, **options) + return _wrap(adapter, sink, secrets) -def build_matrix(store, timeout: float | None = None) -> MatrixAdapter | None: +def build_matrix(store, timeout: float | None = None, sink: OutboundSink | None = None, + secrets: Callable[[], list[str]] = lambda: ()) -> MatrixAdapter | None: if store.get(MATRIX_ENABLED) != FLAG_ON: return None homeserver, user_id = store.get(MATRIX_HOMESERVER), store.get(MATRIX_USER_ID) @@ -26,14 +37,17 @@ def build_matrix(store, timeout: float | None = None) -> MatrixAdapter | None: if not (homeserver and user_id and token and room_id): return None options = {} if timeout is None else {"timeout": timeout} - return MatrixAdapter(homeserver, user_id, token, room_id, **options) + adapter = MatrixAdapter(homeserver, user_id, token, room_id, **options) + return _wrap(adapter, sink, secrets) FACTORIES: tuple[Callable[[object], ChatAdapter | None], ...] = (build_telegram, build_matrix) -def build_adapters(store) -> list[ChatAdapter]: - return [adapter for adapter in (factory(store) for factory in FACTORIES) if adapter is not None] +def build_adapters(store, sink: OutboundSink | None = None, + secrets: Callable[[], list[str]] = lambda: ()) -> list[ChatAdapter]: + return [adapter for adapter in (factory(store, sink=sink, secrets=secrets) for factory in FACTORIES) + if adapter is not None] def default_targets(store) -> list[Target]: diff --git a/app/chatops.py b/app/chatops.py index c5d7dc4..3a05dbb 100644 --- a/app/chatops.py +++ b/app/chatops.py @@ -3,14 +3,17 @@ from typing import Callable from .agent_runs import AgentRunRepository, AgentService, configured_secrets from .alerts import AlertRepository -from .chat import BACKEND_MATRIX, BACKEND_TELEGRAM +from .chat import BACKEND_MATRIX, BACKEND_TELEGRAM, build_matrix, build_telegram from .checkmk import CheckmkClient from .commands import (CallbackHandler, CommandDeps, CommandHandler, SettingsReader, command_audit) from .constants import MATRIX_ALLOWLIST, TELEGRAM_ALLOWLIST from .db import Database from .downtime import DowntimeService, PendingStore from .listeners import ListenerState, MatrixListener, TelegramListener +from .listeners.matrix import HTTP_TIMEOUT_SECONDS as MATRIX_HTTP_TIMEOUT_SECONDS +from .listeners.telegram import HTTP_TIMEOUT_SECONDS as TELEGRAM_HTTP_TIMEOUT_SECONDS from .mutes import MuteService +from .outbound import OutboundRepository from .poller import StateStore ALLOWLIST_KEYS = {BACKEND_TELEGRAM: TELEGRAM_ALLOWLIST, BACKEND_MATRIX: MATRIX_ALLOWLIST} @@ -18,10 +21,15 @@ ALLOWLIST_KEYS = {BACKEND_TELEGRAM: TELEGRAM_ALLOWLIST, BACKEND_MATRIX: MATRIX_A class ChatOps: def __init__(self, db: Database, store, state: StateStore, alerts: AlertRepository, mutes: MuteService, - agent_prompt: Callable[[], str | None] | None = None): + agent_prompt: Callable[[], str | None] | None = None, outbound: OutboundRepository | None = None): def allowlist(backend: str) -> str | None: return store.get(ALLOWLIST_KEYS[backend]) if backend in ALLOWLIST_KEYS else None + outbound = outbound or OutboundRepository(db) + + def secrets() -> list[str]: + return configured_secrets(store) + bot_names: dict[str, str] = {} self.agent_runs = AgentRunRepository(db) self.agent = AgentService(store, db, self.agent_runs, state.recent_changes, @@ -33,8 +41,13 @@ class ChatOps: self.handler = CommandHandler(deps, allowlist, audit_log, bot_name=bot_names.get) self.callbacks = CallbackHandler(deps, allowlist, audit_log) listener_state = ListenerState(db) - self.telegram = TelegramListener(store, self.handler, self.callbacks, listener_state, bot_names) - self.matrix = MatrixListener(store, self.handler, listener_state) + self.telegram = TelegramListener( + store, self.handler, self.callbacks, listener_state, bot_names, + build=lambda s: build_telegram(s, timeout=TELEGRAM_HTTP_TIMEOUT_SECONDS, sink=outbound, + secrets=secrets)) + self.matrix = MatrixListener( + store, self.handler, listener_state, + build=lambda s: build_matrix(s, timeout=MATRIX_HTTP_TIMEOUT_SECONDS, sink=outbound, secrets=secrets)) def listener_runners(self): return [self.telegram.run, self.matrix.run] diff --git a/app/cli.py b/app/cli.py index 393b54d..b116df7 100644 --- a/app/cli.py +++ b/app/cli.py @@ -3,17 +3,20 @@ import getpass import os import secrets import sys +import time from pathlib import Path from .crypto import SecretBox, generate_master_key from .db import Database from .fileconfig import MASTER_KEY_FILE, SESSION_SECRET_FILE, load_file_config, load_master_key +from .outbound import OutboundRepository, parse_since from .users import ROLE_ADMIN, UserError, UserStore CLI_ACTOR = "cli" SECRET_FILE_MODE = 0o600 CONFIG_DIR_MODE = 0o700 SESSION_SECRET_BYTES = 48 +MESSAGES_TEXT_PREVIEW = 200 def _write_secret_file(path: Path, content: str) -> bool: @@ -62,6 +65,24 @@ def cmd_create_admin(args: argparse.Namespace) -> int: return 0 +def cmd_messages(args: argparse.Namespace) -> int: + config = load_file_config() + repo = OutboundRepository(Database(config.db_path)) + since = parse_since(args.since) if args.since else None + try: + entries = repo.search(backend=args.backend, query=args.grep, only_errors=args.errors, since=since, + limit=args.limit) + except ValueError as exc: + print(f"Fehler: {exc}", file=sys.stderr) + return 1 + for entry in entries: + iso = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(entry.ts)) + status = "ok" if entry.ok else f"ERR:{entry.error or ''}" + text = entry.text[:MESSAGES_TEXT_PREVIEW].replace("\n", "\\n") + print(f"{iso} {entry.backend} {entry.target} {entry.action} {status} {text}") + return 0 + + def cmd_set_password(args: argparse.Namespace) -> int: users = _user_store() user = users.get_by_name(args.username) @@ -85,11 +106,18 @@ def main(argv: list[str] | None = None) -> int: create.add_argument("username", nargs="?") reset = commands.add_parser("set-password") reset.add_argument("username") + messages = commands.add_parser("messages") + messages.add_argument("--grep", default=None) + messages.add_argument("--backend", default=None) + messages.add_argument("--errors", action="store_true") + messages.add_argument("--since", default=None) + messages.add_argument("--limit", type=int, default=100) args = parser.parse_args(argv) handlers = { "gen-keys": cmd_gen_keys, "create-admin": cmd_create_admin, "set-password": cmd_set_password, + "messages": cmd_messages, } return handlers[args.command](args) diff --git a/app/constants.py b/app/constants.py index 615e720..2f06af9 100644 --- a/app/constants.py +++ b/app/constants.py @@ -85,9 +85,11 @@ RETENTION_AGENT_RUNS_DAYS = "retention.agent_runs_days" RETENTION_STATE_LOG_DAYS = "retention.state_log_days" RETENTION_AUDIT_DAYS = "retention.audit_days" RETENTION_MUTES_DAYS = "retention.mutes_days" +RETENTION_OUTBOUND_DAYS = "retention.outbound_days" DEFAULT_RETENTION_ALERTS_DAYS = 90 DEFAULT_RETENTION_AGENT_RUNS_DAYS = 90 DEFAULT_RETENTION_STATE_LOG_DAYS = 30 DEFAULT_RETENTION_AUDIT_DAYS = 365 DEFAULT_RETENTION_MUTES_DAYS = 7 +DEFAULT_RETENTION_OUTBOUND_DAYS = 30 diff --git a/app/main.py b/app/main.py index 47886f6..b0bc258 100644 --- a/app/main.py +++ b/app/main.py @@ -17,8 +17,8 @@ from .ratelimit import FailureLimiter from .runtime import Runtime from .settings import SettingsStore from .users import UserStore -from .web import (account, agent_pages, alerts, auth, baseline_pages, matrix_section, mutes_pages, pages, - prompts_pages, prompts_test_pages, retention_pages, routing_pages, selftest_pages, +from .web import (account, agent_pages, alerts, auth, baseline_pages, matrix_section, messages_pages, mutes_pages, + pages, prompts_pages, prompts_test_pages, retention_pages, routing_pages, selftest_pages, telegram_section) from .web.deps import LoginRequired, redirect, templates, verify_csrf @@ -93,5 +93,6 @@ def create_app(config: FileConfig | None = None) -> FastAPI: app.include_router(prompts_test_pages.router) app.include_router(retention_pages.router) app.include_router(selftest_pages.router) + app.include_router(messages_pages.router) app.include_router(pages.router) return app diff --git a/app/migrations.py b/app/migrations.py index 8d1ee8c..9369387 100644 --- a/app/migrations.py +++ b/app/migrations.py @@ -179,4 +179,20 @@ MIGRATIONS = [ checks TEXT NOT NULL ); """, + """ + CREATE TABLE outbound_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER NOT NULL, + backend TEXT NOT NULL, + target TEXT NOT NULL, + action TEXT NOT NULL CHECK(action IN ('send','edit')), + message_id TEXT NOT NULL DEFAULT '', + silent INTEGER NOT NULL DEFAULT 0, + ok INTEGER NOT NULL, + error TEXT, + text TEXT NOT NULL + ); + CREATE INDEX outbound_log_ts ON outbound_log(ts); + CREATE INDEX outbound_log_backend_ts ON outbound_log(backend, ts); + """, ] diff --git a/app/outbound/__init__.py b/app/outbound/__init__.py new file mode 100644 index 0000000..09768f7 --- /dev/null +++ b/app/outbound/__init__.py @@ -0,0 +1,4 @@ +from .repository import OutboundEntry, OutboundRepository, escape_like +from .since import parse_since + +__all__ = ["OutboundEntry", "OutboundRepository", "escape_like", "parse_since"] diff --git a/app/outbound/repository.py b/app/outbound/repository.py new file mode 100644 index 0000000..f54202c --- /dev/null +++ b/app/outbound/repository.py @@ -0,0 +1,75 @@ +import time +from dataclasses import dataclass +from typing import Callable + +from ..db import Database + +ESCAPE_CHAR = "\\" + + +def escape_like(value: str) -> str: + return (value.replace(ESCAPE_CHAR, ESCAPE_CHAR * 2).replace("%", ESCAPE_CHAR + "%") + .replace("_", ESCAPE_CHAR + "_")) + + +@dataclass(frozen=True) +class OutboundEntry: + id: int + ts: int + backend: str + target: str + action: str + message_id: str + silent: bool + ok: bool + error: str | None + text: str + + +def _row(row) -> OutboundEntry: + return OutboundEntry(row["id"], row["ts"], row["backend"], row["target"], row["action"], row["message_id"], + bool(row["silent"]), bool(row["ok"]), row["error"], row["text"]) + + +class OutboundRepository: + def __init__(self, db: Database, clock: Callable[[], float] = time.time): + self._db = db + self._clock = clock + + def record(self, ts: int, backend: str, target: str, action: str, message_id: str, silent: bool, ok: bool, + error: str | None, text: str) -> None: + self._db.insert( + "INSERT INTO outbound_log(ts, backend, target, action, message_id, silent, ok, error, text) " + "VALUES (?,?,?,?,?,?,?,?,?)", + (ts, backend, target, action, message_id, int(bool(silent)), int(bool(ok)), error, text)) + + def _where(self, backend: str | None, query: str | None, only_errors: bool, + since: int | None) -> tuple[str, tuple]: + clauses, params = [], [] + if backend: + clauses.append("backend = ?") + params.append(backend) + if query: + like = f"%{escape_like(query)}%" + clauses.append("(text LIKE ? ESCAPE '\\' OR target LIKE ? ESCAPE '\\')") + params.extend([like, like]) + if only_errors: + clauses.append("ok = 0") + if since is not None: + clauses.append("ts >= ?") + params.append(since) + where = f" WHERE {' AND '.join(clauses)}" if clauses else "" + return where, tuple(params) + + def search(self, backend: str | None = None, query: str | None = None, only_errors: bool = False, + since: int | None = None, limit: int = 100, offset: int = 0) -> list[OutboundEntry]: + where, params = self._where(backend, query, only_errors, since) + rows = self._db.query( + f"SELECT * FROM outbound_log{where} ORDER BY id DESC LIMIT ? OFFSET ?", + params + (limit, offset)) + return [_row(row) for row in rows] + + def count(self, backend: str | None = None, query: str | None = None, only_errors: bool = False, + since: int | None = None) -> int: + where, params = self._where(backend, query, only_errors, since) + return self._db.query(f"SELECT COUNT(*) AS n FROM outbound_log{where}", params)[0]["n"] diff --git a/app/outbound/since.py b/app/outbound/since.py new file mode 100644 index 0000000..f49ef85 --- /dev/null +++ b/app/outbound/since.py @@ -0,0 +1,14 @@ +import re +import time +from typing import Callable + +SINCE_UNITS = {"s": 1, "m": 60, "h": 3600, "d": 86400} +SINCE_RE = re.compile(r"^(\d+)([smhd])$") + + +def parse_since(raw: str, clock: Callable[[], float] = time.time) -> int: + match = SINCE_RE.match(raw.strip()) + if not match: + raise ValueError(f"Ungültiges since: {raw!r} (Beispiel: 2h, 1d)") + amount, unit = match.groups() + return int(clock()) - int(amount) * SINCE_UNITS[unit] diff --git a/app/retention/policy.py b/app/retention/policy.py index bb53218..c1c3966 100644 --- a/app/retention/policy.py +++ b/app/retention/policy.py @@ -2,9 +2,10 @@ from dataclasses import dataclass from typing import Callable from ..constants import (DEFAULT_RETENTION_AGENT_RUNS_DAYS, DEFAULT_RETENTION_ALERTS_DAYS, - DEFAULT_RETENTION_AUDIT_DAYS, DEFAULT_RETENTION_MUTES_DAYS, + DEFAULT_RETENTION_AUDIT_DAYS, DEFAULT_RETENTION_MUTES_DAYS, DEFAULT_RETENTION_OUTBOUND_DAYS, DEFAULT_RETENTION_STATE_LOG_DAYS, RETENTION_AGENT_RUNS_DAYS, RETENTION_ALERTS_DAYS, - RETENTION_AUDIT_DAYS, RETENTION_MUTES_DAYS, RETENTION_STATE_LOG_DAYS) + RETENTION_AUDIT_DAYS, RETENTION_MUTES_DAYS, RETENTION_OUTBOUND_DAYS, + RETENTION_STATE_LOG_DAYS) DAY_SECONDS = 86400 MIN_DAYS = 1 @@ -34,6 +35,9 @@ SETTINGS = ( RetentionSetting(RETENTION_MUTES_DAYS, "Abgelaufene Stummschaltungen", "Tage nach dem Ablauf einer Stummschaltung", DEFAULT_RETENTION_MUTES_DAYS, maximum=MAX_MUTES_DAYS), + RetentionSetting(RETENTION_OUTBOUND_DAYS, "Ausgangslog", + "Protokollierte Telegram-/Matrix-Nachrichten (gesendet und editiert)", + DEFAULT_RETENTION_OUTBOUND_DAYS), ) @@ -44,6 +48,7 @@ class RetentionPolicy: state_log_days: int audit_days: int mutes_days: int + outbound_days: int @dataclass(frozen=True) @@ -53,6 +58,7 @@ class Cutoffs: state_log: int audit: int mutes: int + outbound: int def policy_from(get_int: Callable[[str, int, int], int]) -> RetentionPolicy: @@ -65,7 +71,7 @@ def compute_cutoffs(policy: RetentionPolicy, now: int) -> Cutoffs: return now - days * DAY_SECONDS return Cutoffs(before(policy.alerts_days), before(policy.agent_runs_days), before(policy.state_log_days), - before(policy.audit_days), before(policy.mutes_days)) + before(policy.audit_days), before(policy.mutes_days), before(policy.outbound_days)) def validate_values(values: dict[str, str]) -> list[str]: diff --git a/app/retention/repository.py b/app/retention/repository.py index d529d45..b8e960f 100644 --- a/app/retention/repository.py +++ b/app/retention/repository.py @@ -14,7 +14,9 @@ TABLE_AGENT_RUNS = "agent_runs" TABLE_STATE_LOG = "state_log" TABLE_AUDIT = "audit" TABLE_MUTES = "mutes" -COUNT_KEYS = (TABLE_ALERTS, TABLE_DELIVERIES, TABLE_AGENT_RUNS, TABLE_STATE_LOG, TABLE_AUDIT, TABLE_MUTES) +TABLE_OUTBOUND = "outbound_log" +COUNT_KEYS = (TABLE_ALERTS, TABLE_DELIVERIES, TABLE_AGENT_RUNS, TABLE_STATE_LOG, TABLE_AUDIT, TABLE_MUTES, + TABLE_OUTBOUND) @dataclass(frozen=True) @@ -39,6 +41,7 @@ class RetentionRepository: counts[TABLE_STATE_LOG] = self._purge_table(TABLE_STATE_LOG, "ts", cutoffs.state_log) counts[TABLE_AUDIT] = self._purge_table(TABLE_AUDIT, "ts", cutoffs.audit) counts[TABLE_MUTES] = self._purge_table(TABLE_MUTES, "expires_at", cutoffs.mutes) + counts[TABLE_OUTBOUND] = self._purge_table(TABLE_OUTBOUND, "ts", cutoffs.outbound) return counts def _purge_table(self, table: str, column: str, cutoff: int) -> int: diff --git a/app/runtime.py b/app/runtime.py index e04f026..71c26a0 100644 --- a/app/runtime.py +++ b/app/runtime.py @@ -8,7 +8,7 @@ from . import audit from .alerts import (AlertActions, AlertAnalyzer, AlertPipeline, AlertRepository, SendLimiter, SlidingWindowLimiter, SystemNotifier, base_url_provider, content_mode_provider, send_limit_provider) -from .chat import AdapterRegistry +from .chat import AdapterRegistry, build_adapters from .chatops import ChatOps from .constants import (ALERTS_BUNDLE_SECONDS, ALERTS_RECOVERY_ENABLED, ALERTS_REMINDER_HOURS, DEFAULT_BUNDLE_SECONDS, DEFAULT_REMINDER_HOURS, FLAG_OFF, FLAG_ON, MAX_REMINDER_HOURS, SELFTEST_ENABLED, @@ -17,6 +17,7 @@ from .db import Database from .agent_runs import configured_secrets from .mutes import MuteService from .ollama import OllamaClient +from .outbound import OutboundRepository from .poller import BaselineGuard, Poller, PollerStatus, StateStore from .prompts import (PROMPT_AGENT, PROMPT_ALERT, PromptRepository, PromptService, PromptTester, UserRateLimiter, default_registry) @@ -32,7 +33,9 @@ class Runtime: self.status = PollerStatus() self.alerts = AlertRepository(db) self.state = StateStore(db) - self.registry = AdapterRegistry(store) + self.outbound = OutboundRepository(db) + self.registry = AdapterRegistry( + store, factory=lambda s: build_adapters(s, sink=self.outbound, secrets=lambda: configured_secrets(s))) self.mutes = MuteService(db) self.routes = RouteService(db) self.prompts = PromptService(PromptRepository(db), default_registry(), audit.recorder(db), store.get) @@ -69,7 +72,7 @@ class Runtime: self.pipeline.handle, self.state.mark_reminded, ) self.chatops = ChatOps(db, store, self.state, self.alerts, self.mutes, - lambda: self.prompts.effective_text(PROMPT_AGENT)) + lambda: self.prompts.effective_text(PROMPT_AGENT), outbound=self.outbound) self.prompt_tester = PromptTester( store, lambda: OllamaClient.from_settings(store), self.state.recent_changes, self.chatops.agent, lambda: configured_secrets(store), UserRateLimiter()) diff --git a/app/web/messages_pages.py b/app/web/messages_pages.py new file mode 100644 index 0000000..cef969e --- /dev/null +++ b/app/web/messages_pages.py @@ -0,0 +1,34 @@ +from fastapi import APIRouter, Depends, Request + +from ..chat import BACKEND_MATRIX, BACKEND_TELEGRAM +from ..outbound import parse_since +from ..users import User +from .deps import current_user, render + +router = APIRouter() + +PATH = "/messages" +PAGE_SIZE = 50 +BACKENDS = (BACKEND_TELEGRAM, BACKEND_MATRIX) + + +@router.get(PATH) +def messages_page(request: Request, user: User = Depends(current_user), backend: str = "", q: str = "", + errors: bool = False, since: str = "", page: int = 1): + repo = request.app.state.runtime.outbound + page = max(page, 1) + backend = backend if backend in BACKENDS else "" + since_ts = None + since_error = "" + if since: + try: + since_ts = parse_since(since) + except ValueError: + since_error = f"Ungültiger Zeitraum: {since!r} (Beispiel: 2h, 1d)" + entries = [] if since_error else repo.search(backend=backend or None, query=q or None, only_errors=errors, + since=since_ts, limit=PAGE_SIZE, offset=(page - 1) * PAGE_SIZE) + total = 0 if since_error else repo.count(backend=backend or None, query=q or None, only_errors=errors, + since=since_ts) + return render(request, "messages.html", user=user, entries=entries, backend=backend, q=q, errors=errors, + since=since, since_error=since_error, page=page, page_size=PAGE_SIZE, total=total, + backends=BACKENDS, has_more=page * PAGE_SIZE < total) diff --git a/app/web/retention_section.py b/app/web/retention_section.py index ed728d7..840d58e 100644 --- a/app/web/retention_section.py +++ b/app/web/retention_section.py @@ -10,6 +10,7 @@ COUNT_LABELS = { "state_log": "Statusverlauf", "audit": "Audit-Einträge", "mutes": "Stummschaltungen", + "outbound_log": "Ausgangslog-Einträge", } diff --git a/app/web/templates/base.html b/app/web/templates/base.html index 74a6596..1787ad9 100644 --- a/app/web/templates/base.html +++ b/app/web/templates/base.html @@ -20,6 +20,7 @@ Stummschaltungen Prompts Selbsttest + Ausgangslog {% if user.is_admin %} Setup Checkmk diff --git a/app/web/templates/messages.html b/app/web/templates/messages.html new file mode 100644 index 0000000..71a4b50 --- /dev/null +++ b/app/web/templates/messages.html @@ -0,0 +1,48 @@ +{% extends "base.html" %} +{% block title %}Ausgangslog{% endblock %} +{% block content %} +

Ausgangslog

+
+

Protokolliert jede Nachricht, die der Bot an Telegram oder Matrix sendet oder editiert (Alarme, Befehlsantworten, Systemmeldungen), so wie sie an den jeweiligen Dienst ging. Geheimnisse sind maskiert.

+
+ + + + +
+
+ {% if since_error %}

{{ since_error }}

{% endif %} +
+ + + + {% for e in entries %} + + + + + + + + + {% else %} + + {% endfor %} + +
ZeitBackendZielAktionStatusText
{{ e.ts | fmt_ts }}{{ e.backend }}{{ e.target }}{{ e.action }}{% if e.silent %} leise{% endif %} + {% if e.ok %}ok + {% else %}Fehler
{{ e.error }}{% endif %} +
{{ e.text[:80] }}{% if e.text|length > 80 %}…{% endif %}
{{ e.text }}
Keine Einträge.
+
+

{{ total }} Treffer, Seite {{ page }}.

+
+ {% if page > 1 %}Zurück{% endif %} + {% if has_more %}Weiter{% endif %} +
+
+{% endblock %} diff --git a/deploy/checkbot-cli b/deploy/checkbot-cli index 8eeae33..9c6561e 100755 --- a/deploy/checkbot-cli +++ b/deploy/checkbot-cli @@ -7,7 +7,7 @@ SELF=$(readlink -f "${BASH_SOURCE[0]}") source "$(dirname "$SELF")/lib/common.sh" require_root -[[ $# -gt 0 ]] || die "Aufruf: checkbot-cli [Argumente]" +[[ $# -gt 0 ]] || die "Aufruf: checkbot-cli [Argumente]" cd "$APP_DIR" exec runuser -u "$APP_USER" -- env \ CHECKBOT_CONFIG_DIR="$CONF_DIR" CHECKBOT_DB="$(db_file)" PYTHONDONTWRITEBYTECODE=1 \ diff --git a/tests/test_cli_messages.py b/tests/test_cli_messages.py new file mode 100644 index 0000000..7f75775 --- /dev/null +++ b/tests/test_cli_messages.py @@ -0,0 +1,42 @@ +from app.cli import main +from app.db import Database + + +def seed(db_path): + db = Database(db_path) + db.insert("INSERT INTO outbound_log(ts, backend, target, action, message_id, silent, ok, error, text) " + "VALUES (?,?,?,?,?,?,?,?,?)", (100, "telegram", "1", "send", "a", 0, 1, None, "erste\nzeile")) + db.insert("INSERT INTO outbound_log(ts, backend, target, action, message_id, silent, ok, error, text) " + "VALUES (?,?,?,?,?,?,?,?,?)", (200, "matrix", "!room", "send", "", 0, 0, "boom", "zweite")) + db.close() + + +def test_messages_prints_newest_first(tmp_path, monkeypatch, capsys): + db_path = tmp_path / "checkbot.db" + monkeypatch.setenv("CHECKBOT_CONFIG_DIR", str(tmp_path)) + monkeypatch.setenv("CHECKBOT_DB", str(db_path)) + seed(db_path) + assert main(["messages"]) == 0 + out = capsys.readouterr().out.splitlines() + assert len(out) == 2 + assert "matrix" in out[0] and "ERR:boom" in out[0] + assert "telegram" in out[1] and " ok " in out[1] and "erste\\nzeile" in out[1] + + +def test_messages_filters_errors_only(tmp_path, monkeypatch, capsys): + db_path = tmp_path / "checkbot.db" + monkeypatch.setenv("CHECKBOT_CONFIG_DIR", str(tmp_path)) + monkeypatch.setenv("CHECKBOT_DB", str(db_path)) + seed(db_path) + assert main(["messages", "--errors"]) == 0 + out = capsys.readouterr().out.splitlines() + assert len(out) == 1 and "matrix" in out[0] + + +def test_messages_rejects_invalid_since(tmp_path, monkeypatch, capsys): + db_path = tmp_path / "checkbot.db" + monkeypatch.setenv("CHECKBOT_CONFIG_DIR", str(tmp_path)) + monkeypatch.setenv("CHECKBOT_DB", str(db_path)) + seed(db_path) + assert main(["messages", "--since", "xyz"]) == 1 + assert "Ungültiges" in capsys.readouterr().err diff --git a/tests/test_outbound_logging_adapter.py b/tests/test_outbound_logging_adapter.py new file mode 100644 index 0000000..4bc8c91 --- /dev/null +++ b/tests/test_outbound_logging_adapter.py @@ -0,0 +1,113 @@ +import pytest + +from app.chat import LoggingAdapter, MessageRef +from app.chat.base import ChatError + + +class FakeSink: + def __init__(self): + self.calls = [] + + def record(self, ts, backend, target, action, message_id, silent, ok, error, text): + self.calls.append((ts, backend, target, action, message_id, silent, ok, error, text)) + + +class FakeAdapter: + name = "telegram" + default_target = "1" + + def __init__(self, fail=None): + self.fail = fail + self.sent = [] + + async def send(self, target, text, buttons=(), silent=False): + self.sent.append((target, text, silent)) + if self.fail: + raise self.fail + return MessageRef(self.name, target, "77") + + async def edit(self, ref, text, buttons=(), silent=False): + if self.fail: + raise self.fail + + async def check(self): + return "ok" + + async def aclose(self): + pass + + +class FakeMatrixAdapter(FakeAdapter): + name = "matrix" + + async def reply(self, target, text, html, event_id): + if self.fail: + raise self.fail + return "$ev2" + + +async def test_send_ok_is_recorded(): + sink = FakeSink() + adapter = LoggingAdapter(FakeAdapter(), sink, lambda: [], clock=lambda: 1000.0) + ref = await adapter.send("1", "hallo") + assert ref.message_id == "77" + assert sink.calls == [(1000, "telegram", "1", "send", "77", False, True, None, "hallo")] + + +async def test_send_failure_is_recorded_and_reraised(): + sink = FakeSink() + adapter = LoggingAdapter(FakeAdapter(fail=ChatError("boom")), sink, lambda: [], clock=lambda: 1000.0) + with pytest.raises(ChatError): + await adapter.send("1", "hallo") + ts, backend, target, action, message_id, silent, ok, error, text = sink.calls[0] + assert ok is False and message_id == "" and "boom" in error and text == "hallo" + + +async def test_edit_ok_uses_ref_target_and_message_id(): + sink = FakeSink() + adapter = LoggingAdapter(FakeAdapter(), sink, lambda: [], clock=lambda: 1000.0) + ref = MessageRef("telegram", "42", "99") + await adapter.edit(ref, "neu", silent=True) + assert sink.calls == [(1000, "telegram", "42", "edit", "99", True, True, None, "neu")] + + +async def test_reply_is_logged_as_send_for_matrix(): + sink = FakeSink() + adapter = LoggingAdapter(FakeMatrixAdapter(), sink, lambda: [], clock=lambda: 1000.0) + event_id = await adapter.reply("!room", "antwort", None, "$orig") + assert event_id == "$ev2" + assert sink.calls == [(1000, "matrix", "!room", "send", "$ev2", False, True, None, "antwort")] + + +async def test_logging_failure_does_not_prevent_send(): + class BrokenSink: + def record(self, *args, **kwargs): + raise RuntimeError("db kaputt") + + adapter = LoggingAdapter(FakeAdapter(), BrokenSink(), lambda: []) + ref = await adapter.send("1", "hallo") + assert ref.message_id == "77" + + +async def test_secret_is_masked_in_text_and_error(): + sink = FakeSink() + adapter = LoggingAdapter(FakeAdapter(fail=ChatError("token geheim123 drin")), sink, lambda: ["geheim123"], + clock=lambda: 1000.0) + with pytest.raises(ChatError): + await adapter.send("1", "nachricht mit geheim123") + _, _, _, _, _, _, _, error, text = sink.calls[0] + assert "geheim123" not in error and "geheim123" not in text + assert "***" in error and "***" in text + + +async def test_text_is_truncated_to_4000_chars(): + sink = FakeSink() + adapter = LoggingAdapter(FakeAdapter(), sink, lambda: [], clock=lambda: 1000.0) + await adapter.send("1", "x" * 5000) + assert len(sink.calls[0][-1]) == 4000 + + +def test_unknown_attribute_is_passed_through(): + adapter = LoggingAdapter(FakeAdapter(), FakeSink(), lambda: []) + assert adapter.default_target == "1" + assert adapter.name == "telegram" diff --git a/tests/test_outbound_repository.py b/tests/test_outbound_repository.py new file mode 100644 index 0000000..df4f39b --- /dev/null +++ b/tests/test_outbound_repository.py @@ -0,0 +1,73 @@ +import pytest + +from app.db import Database +from app.outbound import OutboundRepository, escape_like +from app.outbound.since import parse_since + + +def test_escape_like_escapes_wildcards(): + assert escape_like("100%_done\\") == "100\\%\\_done\\\\" + + +def make_repo(): + return OutboundRepository(Database(":memory:")) + + +def test_record_and_search_newest_first(): + repo = make_repo() + repo.record(100, "telegram", "1", "send", "a", False, True, None, "erste") + repo.record(200, "matrix", "!room", "send", "b", False, False, "TelegramError: x", "zweite") + entries = repo.search() + assert [e.text for e in entries] == ["zweite", "erste"] + assert entries[0].ok is False and entries[0].error == "TelegramError: x" + assert entries[1].silent is False + + +def test_search_filters_by_backend(): + repo = make_repo() + repo.record(1, "telegram", "1", "send", "a", False, True, None, "x") + repo.record(2, "matrix", "!room", "send", "b", False, True, None, "y") + assert [e.backend for e in repo.search(backend="matrix")] == ["matrix"] + + +def test_search_filters_only_errors(): + repo = make_repo() + repo.record(1, "telegram", "1", "send", "a", False, True, None, "ok-text") + repo.record(2, "telegram", "1", "send", "", False, False, "boom", "fail-text") + results = repo.search(only_errors=True) + assert len(results) == 1 and results[0].text == "fail-text" + + +def test_search_query_matches_text_or_target_with_escaped_wildcards(): + repo = make_repo() + repo.record(1, "telegram", "host%1", "send", "a", False, True, None, "normal text") + repo.record(2, "telegram", "other", "send", "b", False, True, None, "has host%1 literally") + assert len(repo.search(query="host%1")) == 2 + assert repo.search(query="_") == [] + + +def test_search_since_and_limit_offset(): + repo = make_repo() + for ts in range(1, 6): + repo.record(ts, "telegram", "1", "send", str(ts), False, True, None, f"m{ts}") + assert [e.ts for e in repo.search(since=3)] == [5, 4, 3] + assert [e.ts for e in repo.search(limit=2, offset=1)] == [4, 3] + + +def test_count_matches_search_filters(): + repo = make_repo() + repo.record(1, "telegram", "1", "send", "a", False, True, None, "x") + repo.record(2, "matrix", "!room", "send", "b", False, False, "err", "y") + assert repo.count() == 2 + assert repo.count(only_errors=True) == 1 + assert repo.count(backend="matrix") == 1 + + +def test_parse_since_units(): + assert parse_since("2h", clock=lambda: 1000.0) == 1000 - 2 * 3600 + assert parse_since("1d", clock=lambda: 1000.0) == 1000 - 86400 + + +def test_parse_since_rejects_invalid(): + with pytest.raises(ValueError): + parse_since("abc") diff --git a/tests/test_retention.py b/tests/test_retention.py index ad6fcfb..9920d7e 100644 --- a/tests/test_retention.py +++ b/tests/test_retention.py @@ -7,7 +7,7 @@ import pytest from app import audit from app.constants import (RETENTION_ALERTS_DAYS, RETENTION_AUDIT_DAYS, RETENTION_MUTES_DAYS, - RETENTION_STATE_LOG_DAYS) + RETENTION_OUTBOUND_DAYS, RETENTION_STATE_LOG_DAYS) from app.retention import (COUNT_KEYS, Cutoffs, RetentionBusy, RetentionPolicy, RetentionRepository, RetentionRunner, RetentionService, RunRecord, compute_cutoffs, policy_from, should_run, validate_values) @@ -36,6 +36,10 @@ def seed(db): db.insert("INSERT INTO service_states(host, service, state, last_state_change) VALUES (?,?,?,?)", ("h", "s", 2, OLD)) db.insert("INSERT INTO listener_state(key, value) VALUES (?,?)", ("telegram.offset", "5")) + for ts in (OLD, NEW): + db.insert( + "INSERT INTO outbound_log(ts, backend, target, action, message_id, silent, ok, error, text) " + "VALUES (?,?,?,?,?,?,?,?,?)", (ts, "telegram", "1", "send", "1", 0, 1, None, "x")) def count(db, table): @@ -43,14 +47,15 @@ def count(db, table): def cutoffs_between(): - return Cutoffs(*([NOW - DAY] * 5)) + return Cutoffs(*([NOW - DAY] * 6)) def test_cutoffs_from_defaults(): policy = policy_from(lambda key, default, minimum: default) - assert policy == RetentionPolicy(90, 90, 30, 365, 7) + assert policy == RetentionPolicy(90, 90, 30, 365, 7, 30) cutoffs = compute_cutoffs(policy, NOW) - assert cutoffs == Cutoffs(NOW - 90 * DAY, NOW - 90 * DAY, NOW - 30 * DAY, NOW - 365 * DAY, NOW - 7 * DAY) + assert cutoffs == Cutoffs(NOW - 90 * DAY, NOW - 90 * DAY, NOW - 30 * DAY, NOW - 365 * DAY, NOW - 7 * DAY, + NOW - 30 * DAY) def test_policy_from_store_clamps_and_never_disables(): @@ -59,9 +64,11 @@ def test_policy_from_store_clamps_and_never_disables(): store.set(RETENTION_AUDIT_DAYS, "999999", "t") store.set(RETENTION_STATE_LOG_DAYS, "abc", "t") store.set(RETENTION_MUTES_DAYS, "-5", "t") + store.set(RETENTION_OUTBOUND_DAYS, "999999", "t") policy = policy_from(store.get_int) assert policy.alerts_days == 1 and policy.audit_days == 3650 assert policy.state_log_days == 30 and policy.mutes_days == 1 and policy.agent_runs_days == 90 + assert policy.outbound_days == 3650 def test_validate_values(): @@ -76,10 +83,11 @@ def test_purge_deletes_per_table_with_counts(): db, _, _, _ = make_env() seed(db) counts = RetentionRepository(db).purge(cutoffs_between()) - assert counts == {"alerts": 2, "deliveries": 4, "agent_runs": 1, "state_log": 1, "audit": 1, "mutes": 1} + assert counts == {"alerts": 2, "deliveries": 4, "agent_runs": 1, "state_log": 1, "audit": 1, "mutes": 1, + "outbound_log": 1} assert set(counts) == set(COUNT_KEYS) assert (count(db, "alerts"), count(db, "deliveries"), count(db, "agent_runs")) == (1, 2, 1) - assert (count(db, "state_log"), count(db, "mutes")) == (1, 1) + assert (count(db, "state_log"), count(db, "mutes"), count(db, "outbound_log")) == (1, 1, 1) assert db.query("SELECT COUNT(*) FROM audit WHERE ts=?", (OLD,))[0][0] == 0 @@ -94,15 +102,16 @@ def test_cascade_removes_deliveries_of_deleted_alerts_only(): def test_service_states_and_listener_state_are_never_deleted(): db, _, _, _ = make_env() seed(db) - RetentionRepository(db).purge(Cutoffs(*([NOW + 10 * DAY] * 5))) + RetentionRepository(db).purge(Cutoffs(*([NOW + 10 * DAY] * 6))) assert (count(db, "service_states"), count(db, "listener_state")) == (1, 1) assert (count(db, "alerts"), count(db, "mutes"), count(db, "audit")) == (0, 0, 0) + assert count(db, "outbound_log") == 0 def test_boundary_row_is_kept(): db, _, _, _ = make_env() db.insert("INSERT INTO audit(ts, user, action, target) VALUES (?,?,?,?)", (500, "u", "x", "t")) - assert RetentionRepository(db).purge(Cutoffs(*([500] * 5)))["audit"] == 0 + assert RetentionRepository(db).purge(Cutoffs(*([500] * 6)))["audit"] == 0 def test_batching_with_pause_between_batches(): @@ -128,7 +137,8 @@ def test_checkpoint_runs_on_database(): def make_service(db, policy=None, clock=lambda: float(NOW), repo=None): - return RetentionService(repo or RetentionRepository(db), lambda: policy or RetentionPolicy(90, 90, 30, 365, 7), + return RetentionService(repo or RetentionRepository(db), + lambda: policy or RetentionPolicy(90, 90, 30, 365, 7, 30), audit.recorder(db), clock) @@ -136,7 +146,7 @@ def test_run_writes_audit_state_and_log(caplog): db, _, _, _ = make_env() seed(db) db.insert("INSERT INTO audit(ts, user, action, target) VALUES (?,?,?,?)", (1, "u", "ancient", "t")) - service = make_service(db, RetentionPolicy(1, 1, 1, 1, 1)) + service = make_service(db, RetentionPolicy(1, 1, 1, 1, 1, 1)) with caplog.at_level(logging.INFO, logger="app.retention.service"): record = service.run("manual", "alice") assert record.ts == NOW and record.origin == "manual" and record.counts["alerts"] == 2 diff --git a/tests/test_web_messages.py b/tests/test_web_messages.py new file mode 100644 index 0000000..a7610d4 --- /dev/null +++ b/tests/test_web_messages.py @@ -0,0 +1,53 @@ +from tests.test_web import VIEWER, admin, app, client, login_ok, users, viewer # noqa: F401 + +PAGE = "/messages" + + +def seed(app): + repo = app.state.runtime.outbound + repo.record(100, "telegram", "1", "send", "a", False, True, None, "hallo") + repo.record(200, "matrix", "!room", "send", "", False, False, "Fehlertext", "zweite nachricht") + + +def test_page_requires_login(client): + assert client.get(PAGE).status_code == 303 + + +def test_viewer_can_see_page(client, app, viewer): + seed(app) + login_ok(client, VIEWER) + page = client.get(PAGE) + assert page.status_code == 200 and "Ausgangslog" in page.text + assert "zweite nachricht" in page.text + + +def test_html_in_logged_text_is_escaped_not_rendered(client, app, admin): + seed(app) + login_ok(client) + page = client.get(PAGE) + assert "<b>hallo</b>" in page.text + assert "hallo" not in page.text + + +def test_backend_filter(client, app, admin): + seed(app) + login_ok(client) + page = client.get(PAGE, params={"backend": "matrix"}) + assert "zweite nachricht" in page.text + assert "hallo" not in page.text + + +def test_errors_only_filter(client, app, admin): + seed(app) + login_ok(client) + page = client.get(PAGE, params={"errors": "true"}) + assert "Fehlertext" in page.text + assert "hallo" not in page.text + + +def test_invalid_since_shows_error_and_no_entries(client, app, admin): + seed(app) + login_ok(client) + page = client.get(PAGE, params={"since": "nicht-gueltig"}) + assert "Ungültiger Zeitraum" in page.text + assert "zweite nachricht" not in page.text and "hallo" not in page.text diff --git a/tests/test_web_retention.py b/tests/test_web_retention.py index 4d20ff3..6a94d9e 100644 --- a/tests/test_web_retention.py +++ b/tests/test_web_retention.py @@ -1,7 +1,7 @@ import json from app.constants import (RETENTION_AGENT_RUNS_DAYS, RETENTION_ALERTS_DAYS, RETENTION_AUDIT_DAYS, - RETENTION_MUTES_DAYS, RETENTION_STATE_LOG_DAYS) + RETENTION_MUTES_DAYS, RETENTION_OUTBOUND_DAYS, RETENTION_STATE_LOG_DAYS) from app.retention import RetentionBusy from tests.test_web import (VIEWER, admin, app, client, login_ok, post, # noqa: F401 users, viewer) @@ -11,7 +11,7 @@ PAGE = "/settings/retention" def form(**overrides): data = {"retention_alerts_days": "90", "retention_agent_runs_days": "90", "retention_state_log_days": "30", - "retention_audit_days": "365", "retention_mutes_days": "7"} + "retention_audit_days": "365", "retention_mutes_days": "7", "retention_outbound_days": "30"} data.update(overrides) return data @@ -22,6 +22,8 @@ def seed_old_rows(app): db.insert("INSERT INTO state_log(host, service, state, ts) VALUES (?,?,?,?)", ("h", "s", 1, 1)) db.insert("INSERT INTO agent_runs(ts, backend, sender, command, question, answer) VALUES (?,?,?,?,?,?)", (1, "telegram", "1", "ask", "q", "a")) + db.insert("INSERT INTO outbound_log(ts, backend, target, action, message_id, silent, ok, error, text) " + "VALUES (?,?,?,?,?,?,?,?,?)", (1, "telegram", "1", "send", "1", 0, 1, None, "x")) def test_section_is_admin_only(client, viewer): @@ -59,12 +61,12 @@ def test_invalid_values_are_rejected_and_not_saved(client, app, admin): login_ok(client) for key, bad in (("retention_alerts_days", "0"), ("retention_agent_runs_days", "-3"), ("retention_state_log_days", "abc"), ("retention_audit_days", "99999"), - ("retention_mutes_days", "400")): + ("retention_mutes_days", "400"), ("retention_outbound_days", "0")): response = post(client, PAGE, PAGE, **form(**{key: bad})) assert response.status_code == 422 and "nichts versehentlich ausgeschaltet" in response.text, key store = app.state.store for key in (RETENTION_ALERTS_DAYS, RETENTION_AGENT_RUNS_DAYS, RETENTION_STATE_LOG_DAYS, - RETENTION_AUDIT_DAYS, RETENTION_MUTES_DAYS): + RETENTION_AUDIT_DAYS, RETENTION_MUTES_DAYS, RETENTION_OUTBOUND_DAYS): assert store.get(key) is None @@ -76,6 +78,7 @@ def test_run_now_deletes_reports_and_audits(client, app, admin): page = client.get(PAGE) assert "Aufgeräumt. Gelöscht:" in page.text and "Letzter Lauf" in page.text assert "Audit-Einträge: 1" in page.text and "Statusverlauf: 1" in page.text and "manuell" in page.text + assert "Ausgangslog-Einträge: 1" in page.text db = app.state.db assert db.query("SELECT COUNT(*) FROM agent_runs")[0][0] == 0 rows = db.query("SELECT user, new_value FROM audit WHERE action='retention.run'")