119 lines
4.7 KiB
Python
119 lines
4.7 KiB
Python
from app.checkmk import ServiceState
|
|
from app.models import KIND_PROBLEM, KIND_RECOVERY, STATE_OK, SUPPRESS_DOWNTIME
|
|
from app.poller import compute_events
|
|
from app.routing import ALL_KINDS, Route, delay_provider
|
|
|
|
NOW = 100_000
|
|
DEFAULT_MINUTES = 15
|
|
|
|
|
|
def svc(state=2, service="cpu", changed=NOW, downtime=False, output="boom", flapping=False):
|
|
return ServiceState("h1", service, state, True, output, changed, downtime, flapping)
|
|
|
|
|
|
def zpool_rule():
|
|
return Route(1, "ZFS", 1, True, "*", "*zpool*", 1, ALL_KINDS, False, (), delay_minutes=0)
|
|
|
|
|
|
def make_run(rules=()):
|
|
provider = delay_provider(list(rules), DEFAULT_MINUTES)
|
|
|
|
def run(previous, current, now):
|
|
events, rows = compute_events(previous, current, now, provider)
|
|
return events, {row.key: row for row in rows}
|
|
return run
|
|
|
|
|
|
def test_problem_not_reported_before_delay_expires():
|
|
run = make_run()
|
|
events, rows = run({}, [svc(changed=NOW)], NOW + 899)
|
|
assert events == [] and rows[("h1", "cpu")].reported_state == STATE_OK
|
|
|
|
|
|
def test_problem_reported_after_delay_expires():
|
|
run = make_run()
|
|
_, rows = run({}, [svc(changed=NOW)], NOW + 30)
|
|
events, rows = run(rows, [svc(changed=NOW)], NOW + 900)
|
|
assert [(e.kind, e.state) for e in events] == [(KIND_PROBLEM, 2)]
|
|
assert rows[("h1", "cpu")].reported_state == 2 and rows[("h1", "cpu")].reported_at == NOW + 900
|
|
|
|
|
|
def test_zpool_immediately_other_service_after_15_minutes():
|
|
run = make_run([zpool_rule()])
|
|
current = [svc(state=1, service="zpool status", changed=NOW), svc(state=1, service="cpu", changed=NOW)]
|
|
events, rows = run({}, current, NOW + 30)
|
|
assert [e.service for e in events] == ["zpool status"]
|
|
events, _ = run(rows, current, NOW + 900)
|
|
assert [e.service for e in events] == ["cpu"]
|
|
|
|
|
|
def test_worsening_restarts_the_delay():
|
|
run = make_run()
|
|
_, rows = run({}, [svc(state=1, changed=NOW)], NOW + 900)
|
|
worse = [svc(state=2, changed=NOW + 900)]
|
|
events, rows = run(rows, worse, NOW + 1000)
|
|
assert [e.state for e in events] == []
|
|
events, _ = run(rows, worse, NOW + 900 + 900)
|
|
assert [e.state for e in events] == [2]
|
|
|
|
|
|
def test_problem_gone_before_delay_gives_neither_alert_nor_recovery():
|
|
run = make_run()
|
|
_, rows = run({}, [svc(changed=NOW)], NOW + 60)
|
|
events, rows = run(rows, [], NOW + 120)
|
|
assert events == [] and rows == {}
|
|
|
|
|
|
def test_downtime_during_wait_suppresses_event_at_expiry():
|
|
run = make_run()
|
|
_, rows = run({}, [svc(changed=NOW)], NOW + 60)
|
|
events, _ = run(rows, [svc(changed=NOW, downtime=True)], NOW + 900)
|
|
assert len(events) == 1 and events[0].suppressed and events[0].suppress_reason == SUPPRESS_DOWNTIME
|
|
|
|
|
|
def test_downtime_ended_before_expiry_reports_normally_without_catch_up():
|
|
run = make_run()
|
|
_, rows = run({}, [svc(changed=NOW, downtime=True)], NOW + 60)
|
|
events, rows = run(rows, [svc(changed=NOW)], NOW + 600)
|
|
assert events == []
|
|
events, _ = run(rows, [svc(changed=NOW)], NOW + 900)
|
|
assert len(events) == 1 and not events[0].suppressed
|
|
|
|
|
|
def test_problem_since_survives_warn_crit_changes():
|
|
run = make_run([zpool_rule()])
|
|
current = [svc(state=1, service="zpool", changed=NOW - 7200)]
|
|
_, rows = run({}, current, NOW)
|
|
assert rows[("h1", "zpool")].problem_since == NOW - 7200
|
|
events, rows = run(rows, [svc(state=2, service="zpool", changed=NOW + 30)], NOW + 60)
|
|
assert rows[("h1", "zpool")].problem_since == NOW - 7200
|
|
assert events[0].problem_since_ts == NOW - 7200 and events[0].since_ts == NOW + 30
|
|
|
|
|
|
def test_transition_event_carries_previous_state_and_duration():
|
|
run = make_run([zpool_rule()])
|
|
_, rows = run({}, [svc(state=1, service="zpool", changed=NOW - 7200)], NOW)
|
|
events, _ = run(rows, [svc(state=2, service="zpool", changed=NOW)], NOW + 30)
|
|
assert events[0].previous_state == 1 and events[0].previous_duration == 7200
|
|
|
|
|
|
def test_previous_duration_known_although_delay_phase_updated_the_row():
|
|
run = make_run()
|
|
_, rows = run({}, [svc(state=1, changed=NOW - 7200)], NOW)
|
|
_, rows = run(rows, [svc(state=2, changed=NOW)], NOW + 30)
|
|
events, _ = run(rows, [svc(state=2, changed=NOW)], NOW + 900)
|
|
assert events[0].previous_state == 1 and events[0].previous_duration == 7200
|
|
|
|
|
|
def test_new_problem_has_no_previous_state():
|
|
events, _ = make_run([zpool_rule()])({}, [svc(service="zpool", changed=NOW)], NOW)
|
|
assert events[0].previous_state is None and events[0].problem_since_ts == NOW
|
|
|
|
|
|
def test_recovery_event_carries_last_state_and_problem_start():
|
|
run = make_run([zpool_rule()])
|
|
_, rows = run({}, [svc(state=2, service="zpool", changed=NOW - 3600)], NOW)
|
|
events, _ = run(rows, [], NOW + 600)
|
|
assert [e.kind for e in events] == [KIND_RECOVERY]
|
|
assert events[0].previous_state == 2 and events[0].problem_since_ts == NOW - 3600
|