package alerting import ( "context" "os" "strings" "testing" "time" "github.com/jackc/pgx/v5/pgxpool" dto "github.com/prometheus/client_model/go" "gitea.perlbach24.de/scripte/nexarch/internal/notify" ) func setupTest(t *testing.T) (*RuleStore, *notify.Dispatcher, *pgxpool.Pool, func()) { t.Helper() adminDSN := os.Getenv("TEST_ADMIN_DSN") if adminDSN == "" { t.Skip("TEST_ADMIN_DSN nicht gesetzt, Integrationstest uebersprungen") } ctx := context.Background() pool, err := pgxpool.New(ctx, adminDSN) if err != nil { t.Fatalf("pool: %v", err) } if _, err := pool.Exec(ctx, ` CREATE TABLE IF NOT EXISTS alert_rules ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), metric_name TEXT NOT NULL, comparison TEXT NOT NULL CHECK (comparison IN ('gt','lt')), threshold DOUBLE PRECISION NOT NULL, label_filters JSONB NOT NULL DEFAULT '{}'::jsonb, recipient TEXT NOT NULL, description TEXT NOT NULL DEFAULT '', created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); CREATE TABLE IF NOT EXISTS alert_debounce_state ( rule_key TEXT PRIMARY KEY, last_fired_at TIMESTAMPTZ NOT NULL ); CREATE TABLE IF NOT EXISTS notification_jobs ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), channel TEXT NOT NULL, recipient TEXT NOT NULL, payload JSONB NOT NULL DEFAULT '{}'::jsonb, status TEXT NOT NULL DEFAULT 'pending', attempts INT NOT NULL DEFAULT 0, max_attempts INT NOT NULL DEFAULT 5, next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), last_error TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); `); err != nil { t.Fatalf("schema: %v", err) } cleanup := func() { _, _ = pool.Exec(ctx, `DELETE FROM alert_rules`) _, _ = pool.Exec(ctx, `DELETE FROM alert_debounce_state`) _, _ = pool.Exec(ctx, `DELETE FROM notification_jobs`) pool.Close() } return NewRuleStore(pool), notify.NewDispatcher(pool), pool, cleanup } func gaugeFamily(name string, labels map[string]string, value float64) *dto.MetricFamily { pairs := make([]*dto.LabelPair, 0, len(labels)) for k, v := range labels { k, v := k, v pairs = append(pairs, &dto.LabelPair{Name: &k, Value: &v}) } n := name return &dto.MetricFamily{ Name: &n, Metric: []*dto.Metric{ {Label: pairs, Gauge: &dto.Gauge{Value: &value}}, }, } } // Akzeptanzkriterium 1 + 2 / Pruefung 1 + 2: Überschreitung löst eine // Benachrichtigung mit vollständigem Inhalt (Metrik/Tenant/Modul) aus. func TestEvaluate_FiresAlertOnThresholdExceeded(t *testing.T) { rules, dispatcher, pool, cleanup := setupTest(t) defer cleanup() ctx := context.Background() rule, err := rules.CreateRule(ctx, Rule{ MetricName: "nexarch_core_error_rate", Comparison: ComparisonGreaterThan, Threshold: 0.05, Recipient: "ops@acme.example", Description: "Fehlerrate zu hoch", }) if err != nil { t.Fatalf("create rule: %v", err) } eval := NewEvaluator(rules, dispatcher, pool, time.Hour) families := []*dto.MetricFamily{ gaugeFamily("nexarch_core_error_rate", map[string]string{"tenant": "acme", "module": "dms"}, 0.12), } fired, err := eval.Evaluate(ctx, families) if err != nil { t.Fatalf("evaluate: %v", err) } if len(fired) != 1 || fired[0].Skipped { t.Fatalf("erwartet genau 1 tatsaechlich ausgeloesten alarm, habe %+v", fired) } if fired[0].RuleID != rule.ID { t.Fatalf("rule id = %q, want %q", fired[0].RuleID, rule.ID) } // Pruefung 2: Benachrichtigungsinhalt vollstaendig (Metrik/Tenant/Modul). var payloadJSON []byte if err := pool.QueryRow(ctx, `SELECT payload FROM notification_jobs LIMIT 1`).Scan(&payloadJSON); err != nil { t.Fatalf("notification_jobs lesen: %v", err) } payload := string(payloadJSON) for _, want := range []string{`"metric"`, `nexarch_core_error_rate`, `"tenant"`, `"acme"`, `"module"`, `"dms"`} { if !strings.Contains(payload, want) { t.Errorf("payload enthaelt nicht %q: %s", want, payload) } } } func TestEvaluate_DoesNotFireBelowThreshold(t *testing.T) { rules, dispatcher, pool, cleanup := setupTest(t) defer cleanup() ctx := context.Background() if _, err := rules.CreateRule(ctx, Rule{ MetricName: "nexarch_core_error_rate", Comparison: ComparisonGreaterThan, Threshold: 0.05, Recipient: "ops@acme.example", }); err != nil { t.Fatalf("create rule: %v", err) } eval := NewEvaluator(rules, dispatcher, pool, time.Hour) families := []*dto.MetricFamily{ gaugeFamily("nexarch_core_error_rate", map[string]string{"tenant": "acme"}, 0.01), } fired, err := eval.Evaluate(ctx, families) if err != nil { t.Fatalf("evaluate: %v", err) } if len(fired) != 0 { t.Fatalf("erwartet keinen alarm unterhalb des schwellwerts, habe %+v", fired) } } // Akzeptanzkriterium 3 / Pruefung 3: anhaltende Überschreitung erzeugt NICHT // bei jeder Messung eine neue Benachrichtigung. func TestEvaluate_DebouncesRepeatedFiring(t *testing.T) { rules, dispatcher, pool, cleanup := setupTest(t) defer cleanup() ctx := context.Background() if _, err := rules.CreateRule(ctx, Rule{ MetricName: "nexarch_core_error_rate", Comparison: ComparisonGreaterThan, Threshold: 0.05, Recipient: "ops@acme.example", }); err != nil { t.Fatalf("create rule: %v", err) } // Langes Debounce-Intervall: der zweite Evaluate-Lauf (simuliert die // naechste Messung bei anhaltend ueberschrittenem Wert) darf keinen // weiteren Job einreihen. eval := NewEvaluator(rules, dispatcher, pool, time.Hour) families := []*dto.MetricFamily{ gaugeFamily("nexarch_core_error_rate", map[string]string{"tenant": "acme"}, 0.5), } first, err := eval.Evaluate(ctx, families) if err != nil { t.Fatalf("erster evaluate-lauf: %v", err) } if len(first) != 1 || first[0].Skipped { t.Fatalf("erster lauf haette feuern muessen, habe %+v", first) } second, err := eval.Evaluate(ctx, families) if err != nil { t.Fatalf("zweiter evaluate-lauf: %v", err) } if len(second) != 1 || !second[0].Skipped { t.Fatalf("zweiter lauf haette gedrosselt werden muessen, habe %+v", second) } var count int if err := pool.QueryRow(ctx, `SELECT count(*) FROM notification_jobs`).Scan(&count); err != nil { t.Fatalf("notification_jobs zaehlen: %v", err) } if count != 1 { t.Fatalf("erwartet genau 1 eingereihten job trotz zwei ueberschreitenden messungen, habe %d", count) } } // Verschiedene Zeitreihen derselben Regel (unterschiedlicher Tenant) werden // unabhaengig voneinander gedrosselt. func TestEvaluate_DebouncesIndependentlyPerLabelSet(t *testing.T) { rules, dispatcher, pool, cleanup := setupTest(t) defer cleanup() ctx := context.Background() if _, err := rules.CreateRule(ctx, Rule{ MetricName: "nexarch_core_error_rate", Comparison: ComparisonGreaterThan, Threshold: 0.05, Recipient: "ops@acme.example", }); err != nil { t.Fatalf("create rule: %v", err) } eval := NewEvaluator(rules, dispatcher, pool, time.Hour) families := []*dto.MetricFamily{ { Name: strPtr("nexarch_core_error_rate"), Metric: []*dto.Metric{ metricWithLabel("tenant", "acme", 0.5), metricWithLabel("tenant", "beta", 0.6), }, }, } fired, err := eval.Evaluate(ctx, families) if err != nil { t.Fatalf("evaluate: %v", err) } if len(fired) != 2 { t.Fatalf("erwartet 2 unabhaengige alarme (verschiedene tenants), habe %d", len(fired)) } for _, a := range fired { if a.Skipped { t.Fatalf("beide tenants sollten beim ersten mal feuern, habe %+v", a) } } } func metricWithLabel(name, value string, gaugeValue float64) *dto.Metric { n, v := name, value return &dto.Metric{Label: []*dto.LabelPair{{Name: &n, Value: &v}}, Gauge: &dto.Gauge{Value: &gaugeValue}} } func strPtr(s string) *string { return &s }