OPS-05: alerting-bei-schwellwert-ueberschreitung (internal/alerting: regel-store, evaluator gegen ops-03-metriken, cfg-02-zustellung, drosselung je regel+zeitreihe)

2026-08-28 23:59:09 +02:00
parent 06dbd52d4c
commit 369a40af10
5 changed files with 585 additions and 0 deletions
@@ -0,0 +1,233 @@
+package alerting
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	dto "github.com/prometheus/client_model/go"
+
+	"gitea.perlbach24.de/scripte/nexarch/internal/notify"
+)
+
+func setupTest(t *testing.T) (*RuleStore, *notify.Dispatcher, *pgxpool.Pool, func()) {
+	t.Helper()
+	adminDSN := os.Getenv("TEST_ADMIN_DSN")
+	if adminDSN == "" {
+		t.Skip("TEST_ADMIN_DSN nicht gesetzt, Integrationstest uebersprungen")
+	}
+	ctx := context.Background()
+
+	pool, err := pgxpool.New(ctx, adminDSN)
+	if err != nil {
+		t.Fatalf("pool: %v", err)
+	}
+	if _, err := pool.Exec(ctx, `
+		CREATE TABLE IF NOT EXISTS alert_rules (
+			id UUID PRIMARY KEY DEFAULT gen_random_uuid(), metric_name TEXT NOT NULL,
+			comparison TEXT NOT NULL CHECK (comparison IN ('gt','lt')), threshold DOUBLE PRECISION NOT NULL,
+			label_filters JSONB NOT NULL DEFAULT '{}'::jsonb, recipient TEXT NOT NULL,
+			description TEXT NOT NULL DEFAULT '', created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+		);
+		CREATE TABLE IF NOT EXISTS alert_debounce_state (
+			rule_key TEXT PRIMARY KEY, last_fired_at TIMESTAMPTZ NOT NULL
+		);
+		CREATE TABLE IF NOT EXISTS notification_jobs (
+			id UUID PRIMARY KEY DEFAULT gen_random_uuid(), channel TEXT NOT NULL, recipient TEXT NOT NULL,
+			payload JSONB NOT NULL DEFAULT '{}'::jsonb, status TEXT NOT NULL DEFAULT 'pending', attempts INT NOT NULL DEFAULT 0,
+			max_attempts INT NOT NULL DEFAULT 5, next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), last_error TEXT,
+			created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+		);
+	`); err != nil {
+		t.Fatalf("schema: %v", err)
+	}
+
+	cleanup := func() {
+		_, _ = pool.Exec(ctx, `DELETE FROM alert_rules`)
+		_, _ = pool.Exec(ctx, `DELETE FROM alert_debounce_state`)
+		_, _ = pool.Exec(ctx, `DELETE FROM notification_jobs`)
+		pool.Close()
+	}
+	return NewRuleStore(pool), notify.NewDispatcher(pool), pool, cleanup
+}
+
+func gaugeFamily(name string, labels map[string]string, value float64) *dto.MetricFamily {
+	pairs := make([]*dto.LabelPair, 0, len(labels))
+	for k, v := range labels {
+		k, v := k, v
+		pairs = append(pairs, &dto.LabelPair{Name: &k, Value: &v})
+	}
+	n := name
+	return &dto.MetricFamily{
+		Name: &n,
+		Metric: []*dto.Metric{
+			{Label: pairs, Gauge: &dto.Gauge{Value: &value}},
+		},
+	}
+}
+
+// Akzeptanzkriterium 1 + 2 / Pruefung 1 + 2: Überschreitung löst eine
+// Benachrichtigung mit vollständigem Inhalt (Metrik/Tenant/Modul) aus.
+func TestEvaluate_FiresAlertOnThresholdExceeded(t *testing.T) {
+	rules, dispatcher, pool, cleanup := setupTest(t)
+	defer cleanup()
+	ctx := context.Background()
+
+	rule, err := rules.CreateRule(ctx, Rule{
+		MetricName: "nexarch_core_error_rate", Comparison: ComparisonGreaterThan, Threshold: 0.05,
+		Recipient: "ops@acme.example", Description: "Fehlerrate zu hoch",
+	})
+	if err != nil {
+		t.Fatalf("create rule: %v", err)
+	}
+
+	eval := NewEvaluator(rules, dispatcher, pool, time.Hour)
+	families := []*dto.MetricFamily{
+		gaugeFamily("nexarch_core_error_rate", map[string]string{"tenant": "acme", "module": "dms"}, 0.12),
+	}
+
+	fired, err := eval.Evaluate(ctx, families)
+	if err != nil {
+		t.Fatalf("evaluate: %v", err)
+	}
+	if len(fired) != 1 || fired[0].Skipped {
+		t.Fatalf("erwartet genau 1 tatsaechlich ausgeloesten alarm, habe %+v", fired)
+	}
+	if fired[0].RuleID != rule.ID {
+		t.Fatalf("rule id = %q, want %q", fired[0].RuleID, rule.ID)
+	}
+
+	// Pruefung 2: Benachrichtigungsinhalt vollstaendig (Metrik/Tenant/Modul).
+	var payloadJSON []byte
+	if err := pool.QueryRow(ctx, `SELECT payload FROM notification_jobs LIMIT 1`).Scan(&payloadJSON); err != nil {
+		t.Fatalf("notification_jobs lesen: %v", err)
+	}
+	payload := string(payloadJSON)
+	for _, want := range []string{`"metric"`, `nexarch_core_error_rate`, `"tenant"`, `"acme"`, `"module"`, `"dms"`} {
+		if !strings.Contains(payload, want) {
+			t.Errorf("payload enthaelt nicht %q: %s", want, payload)
+		}
+	}
+}
+
+func TestEvaluate_DoesNotFireBelowThreshold(t *testing.T) {
+	rules, dispatcher, pool, cleanup := setupTest(t)
+	defer cleanup()
+	ctx := context.Background()
+
+	if _, err := rules.CreateRule(ctx, Rule{
+		MetricName: "nexarch_core_error_rate", Comparison: ComparisonGreaterThan, Threshold: 0.05,
+		Recipient: "ops@acme.example",
+	}); err != nil {
+		t.Fatalf("create rule: %v", err)
+	}
+
+	eval := NewEvaluator(rules, dispatcher, pool, time.Hour)
+	families := []*dto.MetricFamily{
+		gaugeFamily("nexarch_core_error_rate", map[string]string{"tenant": "acme"}, 0.01),
+	}
+
+	fired, err := eval.Evaluate(ctx, families)
+	if err != nil {
+		t.Fatalf("evaluate: %v", err)
+	}
+	if len(fired) != 0 {
+		t.Fatalf("erwartet keinen alarm unterhalb des schwellwerts, habe %+v", fired)
+	}
+}
+
+// Akzeptanzkriterium 3 / Pruefung 3: anhaltende Überschreitung erzeugt NICHT
+// bei jeder Messung eine neue Benachrichtigung.
+func TestEvaluate_DebouncesRepeatedFiring(t *testing.T) {
+	rules, dispatcher, pool, cleanup := setupTest(t)
+	defer cleanup()
+	ctx := context.Background()
+
+	if _, err := rules.CreateRule(ctx, Rule{
+		MetricName: "nexarch_core_error_rate", Comparison: ComparisonGreaterThan, Threshold: 0.05,
+		Recipient: "ops@acme.example",
+	}); err != nil {
+		t.Fatalf("create rule: %v", err)
+	}
+
+	// Langes Debounce-Intervall: der zweite Evaluate-Lauf (simuliert die
+	// naechste Messung bei anhaltend ueberschrittenem Wert) darf keinen
+	// weiteren Job einreihen.
+	eval := NewEvaluator(rules, dispatcher, pool, time.Hour)
+	families := []*dto.MetricFamily{
+		gaugeFamily("nexarch_core_error_rate", map[string]string{"tenant": "acme"}, 0.5),
+	}
+
+	first, err := eval.Evaluate(ctx, families)
+	if err != nil {
+		t.Fatalf("erster evaluate-lauf: %v", err)
+	}
+	if len(first) != 1 || first[0].Skipped {
+		t.Fatalf("erster lauf haette feuern muessen, habe %+v", first)
+	}
+
+	second, err := eval.Evaluate(ctx, families)
+	if err != nil {
+		t.Fatalf("zweiter evaluate-lauf: %v", err)
+	}
+	if len(second) != 1 || !second[0].Skipped {
+		t.Fatalf("zweiter lauf haette gedrosselt werden muessen, habe %+v", second)
+	}
+
+	var count int
+	if err := pool.QueryRow(ctx, `SELECT count(*) FROM notification_jobs`).Scan(&count); err != nil {
+		t.Fatalf("notification_jobs zaehlen: %v", err)
+	}
+	if count != 1 {
+		t.Fatalf("erwartet genau 1 eingereihten job trotz zwei ueberschreitenden messungen, habe %d", count)
+	}
+}
+
+// Verschiedene Zeitreihen derselben Regel (unterschiedlicher Tenant) werden
+// unabhaengig voneinander gedrosselt.
+func TestEvaluate_DebouncesIndependentlyPerLabelSet(t *testing.T) {
+	rules, dispatcher, pool, cleanup := setupTest(t)
+	defer cleanup()
+	ctx := context.Background()
+
+	if _, err := rules.CreateRule(ctx, Rule{
+		MetricName: "nexarch_core_error_rate", Comparison: ComparisonGreaterThan, Threshold: 0.05,
+		Recipient: "ops@acme.example",
+	}); err != nil {
+		t.Fatalf("create rule: %v", err)
+	}
+
+	eval := NewEvaluator(rules, dispatcher, pool, time.Hour)
+	families := []*dto.MetricFamily{
+		{
+			Name: strPtr("nexarch_core_error_rate"),
+			Metric: []*dto.Metric{
+				metricWithLabel("tenant", "acme", 0.5),
+				metricWithLabel("tenant", "beta", 0.6),
+			},
+		},
+	}
+
+	fired, err := eval.Evaluate(ctx, families)
+	if err != nil {
+		t.Fatalf("evaluate: %v", err)
+	}
+	if len(fired) != 2 {
+		t.Fatalf("erwartet 2 unabhaengige alarme (verschiedene tenants), habe %d", len(fired))
+	}
+	for _, a := range fired {
+		if a.Skipped {
+			t.Fatalf("beide tenants sollten beim ersten mal feuern, habe %+v", a)
+		}
+	}
+}
+
+func metricWithLabel(name, value string, gaugeValue float64) *dto.Metric {
+	n, v := name, value
+	return &dto.Metric{Label: []*dto.LabelPair{{Name: &n, Value: &v}}, Gauge: &dto.Gauge{Value: &gaugeValue}}
+}
+
+func strPtr(s string) *string { return &s }