checkbot/tests/test_reminders.py

from app.db import Database
from app.models import KIND_REMINDER, StateRow
from app.mutes import SOURCE_UI, Mute
from app.poller import StateStore
from app.reminders import ReminderWorker, due_reminders
from app.routing import ALL_KINDS, Route

NOW = 1_000_000
HOUR = 3600


def row(state=2, reported=2, reported_at=NOW - 5 * HOUR, last_reminder=0, downtime=False, flapping=False,
        service="zpool status"):
    return StateRow("h1", service, state, "kaputt", NOW - 6 * HOUR, downtime, flapping, reported, False,
                    NOW - 7 * HOUR, reported_at, NOW - 6 * HOUR, last_reminder)


def rule(hours=4, host="*", position=1):
    return Route(position, f"r{position}", position, True, host, "", 1, ALL_KINDS, False, (), reminder_hours=hours)


def due(rows, rules=(), mutes=(), default=0, now=NOW):
    return due_reminders(rows, list(rules), list(mutes), default, now)


def test_reminder_when_interval_elapsed_since_report():
    (event,) = due([row()], [rule(4)])
    assert event.kind == KIND_REMINDER and event.state == 2 and event.output == "kaputt"
    assert event.since_ts == NOW - 6 * HOUR and event.problem_since_ts == NOW - 7 * HOUR


def test_no_reminder_before_interval():
    assert due([row(reported_at=NOW - 3 * HOUR)], [rule(4)]) == []


def test_last_reminder_counts_when_newer_than_report():
    assert due([row(last_reminder=NOW - 3 * HOUR)], [rule(4)]) == []
    assert len(due([row(last_reminder=NOW - 4 * HOUR)], [rule(4)])) == 1


def test_only_crit_state_and_reported_crit():
    assert due([row(state=1, reported=1)], [rule(4)]) == []
    assert due([row(state=1, reported=2)], [rule(4)]) == []
    assert due([row(state=2, reported=1)], [rule(4)]) == []
    assert due([row(state=2, reported=0)], [rule(4)]) == []


def test_no_reminder_during_downtime_or_flapping():
    assert due([row(downtime=True)], [rule(4)]) == []
    assert due([row(flapping=True)], [rule(4)]) == []


def test_no_reminder_during_mute():
    mute = Mute(1, "h1", "*zpool*", NOW + HOUR, "", "admin", 0, SOURCE_UI)
    assert due([row()], [rule(4)], [mute]) == []
    expired = Mute(1, "h1", "*zpool*", NOW - 1, "", "admin", 0, SOURCE_UI)
    assert len(due([row()], [rule(4)], [expired])) == 1


def test_smallest_rule_interval_wins_and_overrides_default():
    rules = [rule(8, position=1), rule(2, position=2)]
    assert len(due([row(reported_at=NOW - 2 * HOUR)], rules, default=6)) == 1
    assert due([row(reported_at=NOW - 5 * HOUR)], [rule(8)], default=2) == []


def test_global_default_applies_without_rule_interval():
    assert due([row()], [], default=0) == []
    assert len(due([row()], [], default=4)) == 1
    assert len(due([row()], [rule(None)], default=4)) == 1


def test_rule_with_explicit_zero_turns_reminder_off_despite_default():
    assert due([row()], [rule(0)], default=4) == []
    assert due([row()], [rule(0), rule(4, position=2)], default=4) == []


def test_acknowledged_crit_gets_no_reminder_but_worsened_ack_does():
    from dataclasses import replace
    assert due([replace(row(), ack_state=2)], [rule(4)]) == []
    assert len(due([replace(row(), ack_state=1)], [rule(4)])) == 1


def test_rule_for_other_host_does_not_apply():
    assert due([row()], [rule(4, host="db*")], default=0) == []


class Env:
    def __init__(self, rules=(), mutes=(), default=0):
        self.db = Database(":memory:")
        self.state = StateStore(self.db)
        self.sent = []
        self.clock = [NOW]
        self.rules, self.mutes, self.default = list(rules), list(mutes), default
        self.worker = self.make_worker()

    def make_worker(self):
        async def sink(events):
            self.sent.extend(events)

        return ReminderWorker(self.state.load, lambda: self.rules, lambda now: self.mutes, lambda: self.default,
                              sink, self.state.mark_reminded, clock=lambda: self.clock[0])

    def store(self, state_row):
        self.state.apply({}, [state_row], NOW)


async def test_worker_sends_and_persists_timestamp():
    env = Env([rule(4)])
    env.store(row())
    assert await env.worker.tick() == 1
    assert env.state.load()[("h1", "zpool status")].last_reminder_ts == NOW
    assert len(env.sent) == 1 and env.sent[0].kind == KIND_REMINDER


async def test_worker_waits_full_interval_for_next_reminder():
    env = Env([rule(4)])
    env.store(row())
    await env.worker.tick()
    env.clock[0] = NOW + 3 * HOUR
    assert await env.worker.tick() == 0
    env.clock[0] = NOW + 4 * HOUR
    assert await env.worker.tick() == 1


async def test_no_double_send_after_restart():
    env = Env([rule(4)])
    env.store(row())
    await env.worker.tick()
    env.sent.clear()
    restarted = env.make_worker()
    env.clock[0] = NOW + 60
    assert await restarted.tick() == 0 and env.sent == []


async def test_worker_skips_muted_and_downtime_services():
    mute = Mute(1, "h1", "*", NOW + HOUR, "", "admin", 0, SOURCE_UI)
    env = Env([rule(4)], [mute])
    env.store(row())
    assert await env.worker.tick() == 0
    assert env.state.load()[("h1", "zpool status")].last_reminder_ts == 0


async def test_poller_upsert_keeps_reminder_timestamp():
    env = Env([rule(4)])
    env.store(row())
    await env.worker.tick()
    previous = env.state.load()
    changed = previous[("h1", "zpool status")]
    from dataclasses import replace
    stale = replace(changed, output="neu", last_reminder_ts=0)
    env.state.apply(previous, [stale], NOW + 30)
    assert env.state.load()[("h1", "zpool status")].last_reminder_ts == NOW


async def test_worker_error_in_sink_does_not_stop_run_loop():
    import asyncio
    env = Env([rule(4)])
    env.store(row())
    calls = []

    async def failing_sink(events):
        calls.append(1)
        raise RuntimeError("pipeline kaputt")

    async def sleep(seconds):
        if len(calls) >= 1:
            raise asyncio.CancelledError

    worker = ReminderWorker(env.state.load, lambda: env.rules, lambda now: [], lambda: 0, failing_sink,
                            env.state.mark_reminded, clock=lambda: NOW, sleep=sleep)
    try:
        await worker.run()
    except asyncio.CancelledError:
        pass
    assert calls == [1]