from app.db import Database from app.models import KIND_REMINDER, StateRow from app.mutes import SOURCE_UI, Mute from app.poller import StateStore from app.reminders import ReminderWorker, due_reminders from app.routing import ALL_KINDS, Route NOW = 1_000_000 HOUR = 3600 def row(state=2, reported=2, reported_at=NOW - 5 * HOUR, last_reminder=0, downtime=False, flapping=False, service="zpool status"): return StateRow("h1", service, state, "kaputt", NOW - 6 * HOUR, downtime, flapping, reported, False, NOW - 7 * HOUR, reported_at, NOW - 6 * HOUR, last_reminder) def rule(hours=4, host="*", position=1): return Route(position, f"r{position}", position, True, host, "", 1, ALL_KINDS, False, (), reminder_hours=hours) def due(rows, rules=(), mutes=(), default=0, now=NOW): return due_reminders(rows, list(rules), list(mutes), default, now) def test_reminder_when_interval_elapsed_since_report(): (event,) = due([row()], [rule(4)]) assert event.kind == KIND_REMINDER and event.state == 2 and event.output == "kaputt" assert event.since_ts == NOW - 6 * HOUR and event.problem_since_ts == NOW - 7 * HOUR def test_no_reminder_before_interval(): assert due([row(reported_at=NOW - 3 * HOUR)], [rule(4)]) == [] def test_last_reminder_counts_when_newer_than_report(): assert due([row(last_reminder=NOW - 3 * HOUR)], [rule(4)]) == [] assert len(due([row(last_reminder=NOW - 4 * HOUR)], [rule(4)])) == 1 def test_only_crit_state_and_reported_crit(): assert due([row(state=1, reported=1)], [rule(4)]) == [] assert due([row(state=1, reported=2)], [rule(4)]) == [] assert due([row(state=2, reported=1)], [rule(4)]) == [] assert due([row(state=2, reported=0)], [rule(4)]) == [] def test_no_reminder_during_downtime_or_flapping(): assert due([row(downtime=True)], [rule(4)]) == [] assert due([row(flapping=True)], [rule(4)]) == [] def test_no_reminder_during_mute(): mute = Mute(1, "h1", "*zpool*", NOW + HOUR, "", "admin", 0, SOURCE_UI) assert due([row()], [rule(4)], [mute]) == [] expired = Mute(1, "h1", "*zpool*", NOW - 1, "", "admin", 0, SOURCE_UI) assert len(due([row()], [rule(4)], [expired])) == 1 def test_smallest_rule_interval_wins_and_overrides_default(): rules = [rule(8, position=1), rule(2, position=2)] assert len(due([row(reported_at=NOW - 2 * HOUR)], rules, default=6)) == 1 assert due([row(reported_at=NOW - 5 * HOUR)], [rule(8)], default=2) == [] def test_global_default_applies_without_rule_interval(): assert due([row()], [], default=0) == [] assert len(due([row()], [], default=4)) == 1 assert len(due([row()], [rule(None)], default=4)) == 1 def test_rule_with_explicit_zero_turns_reminder_off_despite_default(): assert due([row()], [rule(0)], default=4) == [] assert due([row()], [rule(0), rule(4, position=2)], default=4) == [] def test_acknowledged_crit_gets_no_reminder_but_worsened_ack_does(): from dataclasses import replace assert due([replace(row(), ack_state=2)], [rule(4)]) == [] assert len(due([replace(row(), ack_state=1)], [rule(4)])) == 1 def test_rule_for_other_host_does_not_apply(): assert due([row()], [rule(4, host="db*")], default=0) == [] class Env: def __init__(self, rules=(), mutes=(), default=0): self.db = Database(":memory:") self.state = StateStore(self.db) self.sent = [] self.clock = [NOW] self.rules, self.mutes, self.default = list(rules), list(mutes), default self.worker = self.make_worker() def make_worker(self): async def sink(events): self.sent.extend(events) return ReminderWorker(self.state.load, lambda: self.rules, lambda now: self.mutes, lambda: self.default, sink, self.state.mark_reminded, clock=lambda: self.clock[0]) def store(self, state_row): self.state.apply({}, [state_row], NOW) async def test_worker_sends_and_persists_timestamp(): env = Env([rule(4)]) env.store(row()) assert await env.worker.tick() == 1 assert env.state.load()[("h1", "zpool status")].last_reminder_ts == NOW assert len(env.sent) == 1 and env.sent[0].kind == KIND_REMINDER async def test_worker_waits_full_interval_for_next_reminder(): env = Env([rule(4)]) env.store(row()) await env.worker.tick() env.clock[0] = NOW + 3 * HOUR assert await env.worker.tick() == 0 env.clock[0] = NOW + 4 * HOUR assert await env.worker.tick() == 1 async def test_no_double_send_after_restart(): env = Env([rule(4)]) env.store(row()) await env.worker.tick() env.sent.clear() restarted = env.make_worker() env.clock[0] = NOW + 60 assert await restarted.tick() == 0 and env.sent == [] async def test_worker_skips_muted_and_downtime_services(): mute = Mute(1, "h1", "*", NOW + HOUR, "", "admin", 0, SOURCE_UI) env = Env([rule(4)], [mute]) env.store(row()) assert await env.worker.tick() == 0 assert env.state.load()[("h1", "zpool status")].last_reminder_ts == 0 async def test_poller_upsert_keeps_reminder_timestamp(): env = Env([rule(4)]) env.store(row()) await env.worker.tick() previous = env.state.load() changed = previous[("h1", "zpool status")] from dataclasses import replace stale = replace(changed, output="neu", last_reminder_ts=0) env.state.apply(previous, [stale], NOW + 30) assert env.state.load()[("h1", "zpool status")].last_reminder_ts == NOW async def test_worker_error_in_sink_does_not_stop_run_loop(): import asyncio env = Env([rule(4)]) env.store(row()) calls = [] async def failing_sink(events): calls.append(1) raise RuntimeError("pipeline kaputt") async def sleep(seconds): if len(calls) >= 1: raise asyncio.CancelledError worker = ReminderWorker(env.state.load, lambda: env.rules, lambda now: [], lambda: 0, failing_sink, env.state.mark_reminded, clock=lambda: NOW, sleep=sleep) try: await worker.run() except asyncio.CancelledError: pass assert calls == [1]