nexarch/internal/alerting/evaluator_test.go

package alerting

import (
	"context"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/jackc/pgx/v5/pgxpool"
	dto "github.com/prometheus/client_model/go"

	"gitea.perlbach24.de/scripte/nexarch/internal/notify"
)

func setupTest(t *testing.T) (*RuleStore, *notify.Dispatcher, *pgxpool.Pool, func()) {
	t.Helper()
	adminDSN := os.Getenv("TEST_ADMIN_DSN")
	if adminDSN == "" {
		t.Skip("TEST_ADMIN_DSN nicht gesetzt, Integrationstest uebersprungen")
	}
	ctx := context.Background()

	pool, err := pgxpool.New(ctx, adminDSN)
	if err != nil {
		t.Fatalf("pool: %v", err)
	}
	if _, err := pool.Exec(ctx, `
		CREATE TABLE IF NOT EXISTS alert_rules (
			id UUID PRIMARY KEY DEFAULT gen_random_uuid(), metric_name TEXT NOT NULL,
			comparison TEXT NOT NULL CHECK (comparison IN ('gt','lt')), threshold DOUBLE PRECISION NOT NULL,
			label_filters JSONB NOT NULL DEFAULT '{}'::jsonb, recipient TEXT NOT NULL,
			description TEXT NOT NULL DEFAULT '', created_at TIMESTAMPTZ NOT NULL DEFAULT now()
		);
		CREATE TABLE IF NOT EXISTS alert_debounce_state (
			rule_key TEXT PRIMARY KEY, last_fired_at TIMESTAMPTZ NOT NULL
		);
		CREATE TABLE IF NOT EXISTS notification_jobs (
			id UUID PRIMARY KEY DEFAULT gen_random_uuid(), channel TEXT NOT NULL, recipient TEXT NOT NULL,
			payload JSONB NOT NULL DEFAULT '{}'::jsonb, status TEXT NOT NULL DEFAULT 'pending', attempts INT NOT NULL DEFAULT 0,
			max_attempts INT NOT NULL DEFAULT 5, next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), last_error TEXT,
			created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
		);
	`); err != nil {
		t.Fatalf("schema: %v", err)
	}

	cleanup := func() {
		_, _ = pool.Exec(ctx, `DELETE FROM alert_rules`)
		_, _ = pool.Exec(ctx, `DELETE FROM alert_debounce_state`)
		_, _ = pool.Exec(ctx, `DELETE FROM notification_jobs`)
		pool.Close()
	}
	return NewRuleStore(pool), notify.NewDispatcher(pool), pool, cleanup
}

func gaugeFamily(name string, labels map[string]string, value float64) *dto.MetricFamily {
	pairs := make([]*dto.LabelPair, 0, len(labels))
	for k, v := range labels {
		k, v := k, v
		pairs = append(pairs, &dto.LabelPair{Name: &k, Value: &v})
	}
	n := name
	return &dto.MetricFamily{
		Name: &n,
		Metric: []*dto.Metric{
			{Label: pairs, Gauge: &dto.Gauge{Value: &value}},
		},
	}
}

// Akzeptanzkriterium 1 + 2 / Pruefung 1 + 2: Überschreitung löst eine
// Benachrichtigung mit vollständigem Inhalt (Metrik/Tenant/Modul) aus.
func TestEvaluate_FiresAlertOnThresholdExceeded(t *testing.T) {
	rules, dispatcher, pool, cleanup := setupTest(t)
	defer cleanup()
	ctx := context.Background()

	rule, err := rules.CreateRule(ctx, Rule{
		MetricName: "nexarch_core_error_rate", Comparison: ComparisonGreaterThan, Threshold: 0.05,
		Recipient: "ops@acme.example", Description: "Fehlerrate zu hoch",
	})
	if err != nil {
		t.Fatalf("create rule: %v", err)
	}

	eval := NewEvaluator(rules, dispatcher, pool, time.Hour)
	families := []*dto.MetricFamily{
		gaugeFamily("nexarch_core_error_rate", map[string]string{"tenant": "acme", "module": "dms"}, 0.12),
	}

	fired, err := eval.Evaluate(ctx, families)
	if err != nil {
		t.Fatalf("evaluate: %v", err)
	}
	if len(fired) != 1 || fired[0].Skipped {
		t.Fatalf("erwartet genau 1 tatsaechlich ausgeloesten alarm, habe %+v", fired)
	}
	if fired[0].RuleID != rule.ID {
		t.Fatalf("rule id = %q, want %q", fired[0].RuleID, rule.ID)
	}

	// Pruefung 2: Benachrichtigungsinhalt vollstaendig (Metrik/Tenant/Modul).
	var payloadJSON []byte
	if err := pool.QueryRow(ctx, `SELECT payload FROM notification_jobs LIMIT 1`).Scan(&payloadJSON); err != nil {
		t.Fatalf("notification_jobs lesen: %v", err)
	}
	payload := string(payloadJSON)
	for _, want := range []string{`"metric"`, `nexarch_core_error_rate`, `"tenant"`, `"acme"`, `"module"`, `"dms"`} {
		if !strings.Contains(payload, want) {
			t.Errorf("payload enthaelt nicht %q: %s", want, payload)
		}
	}
}

func TestEvaluate_DoesNotFireBelowThreshold(t *testing.T) {
	rules, dispatcher, pool, cleanup := setupTest(t)
	defer cleanup()
	ctx := context.Background()

	if _, err := rules.CreateRule(ctx, Rule{
		MetricName: "nexarch_core_error_rate", Comparison: ComparisonGreaterThan, Threshold: 0.05,
		Recipient: "ops@acme.example",
	}); err != nil {
		t.Fatalf("create rule: %v", err)
	}

	eval := NewEvaluator(rules, dispatcher, pool, time.Hour)
	families := []*dto.MetricFamily{
		gaugeFamily("nexarch_core_error_rate", map[string]string{"tenant": "acme"}, 0.01),
	}

	fired, err := eval.Evaluate(ctx, families)
	if err != nil {
		t.Fatalf("evaluate: %v", err)
	}
	if len(fired) != 0 {
		t.Fatalf("erwartet keinen alarm unterhalb des schwellwerts, habe %+v", fired)
	}
}

// Akzeptanzkriterium 3 / Pruefung 3: anhaltende Überschreitung erzeugt NICHT
// bei jeder Messung eine neue Benachrichtigung.
func TestEvaluate_DebouncesRepeatedFiring(t *testing.T) {
	rules, dispatcher, pool, cleanup := setupTest(t)
	defer cleanup()
	ctx := context.Background()

	if _, err := rules.CreateRule(ctx, Rule{
		MetricName: "nexarch_core_error_rate", Comparison: ComparisonGreaterThan, Threshold: 0.05,
		Recipient: "ops@acme.example",
	}); err != nil {
		t.Fatalf("create rule: %v", err)
	}

	// Langes Debounce-Intervall: der zweite Evaluate-Lauf (simuliert die
	// naechste Messung bei anhaltend ueberschrittenem Wert) darf keinen
	// weiteren Job einreihen.
	eval := NewEvaluator(rules, dispatcher, pool, time.Hour)
	families := []*dto.MetricFamily{
		gaugeFamily("nexarch_core_error_rate", map[string]string{"tenant": "acme"}, 0.5),
	}

	first, err := eval.Evaluate(ctx, families)
	if err != nil {
		t.Fatalf("erster evaluate-lauf: %v", err)
	}
	if len(first) != 1 || first[0].Skipped {
		t.Fatalf("erster lauf haette feuern muessen, habe %+v", first)
	}

	second, err := eval.Evaluate(ctx, families)
	if err != nil {
		t.Fatalf("zweiter evaluate-lauf: %v", err)
	}
	if len(second) != 1 || !second[0].Skipped {
		t.Fatalf("zweiter lauf haette gedrosselt werden muessen, habe %+v", second)
	}

	var count int
	if err := pool.QueryRow(ctx, `SELECT count(*) FROM notification_jobs`).Scan(&count); err != nil {
		t.Fatalf("notification_jobs zaehlen: %v", err)
	}
	if count != 1 {
		t.Fatalf("erwartet genau 1 eingereihten job trotz zwei ueberschreitenden messungen, habe %d", count)
	}
}

// Verschiedene Zeitreihen derselben Regel (unterschiedlicher Tenant) werden
// unabhaengig voneinander gedrosselt.
func TestEvaluate_DebouncesIndependentlyPerLabelSet(t *testing.T) {
	rules, dispatcher, pool, cleanup := setupTest(t)
	defer cleanup()
	ctx := context.Background()

	if _, err := rules.CreateRule(ctx, Rule{
		MetricName: "nexarch_core_error_rate", Comparison: ComparisonGreaterThan, Threshold: 0.05,
		Recipient: "ops@acme.example",
	}); err != nil {
		t.Fatalf("create rule: %v", err)
	}

	eval := NewEvaluator(rules, dispatcher, pool, time.Hour)
	families := []*dto.MetricFamily{
		{
			Name: strPtr("nexarch_core_error_rate"),
			Metric: []*dto.Metric{
				metricWithLabel("tenant", "acme", 0.5),
				metricWithLabel("tenant", "beta", 0.6),
			},
		},
	}

	fired, err := eval.Evaluate(ctx, families)
	if err != nil {
		t.Fatalf("evaluate: %v", err)
	}
	if len(fired) != 2 {
		t.Fatalf("erwartet 2 unabhaengige alarme (verschiedene tenants), habe %d", len(fired))
	}
	for _, a := range fired {
		if a.Skipped {
			t.Fatalf("beide tenants sollten beim ersten mal feuern, habe %+v", a)
		}
	}
}

func metricWithLabel(name, value string, gaugeValue float64) *dto.Metric {
	n, v := name, value
	return &dto.Metric{Label: []*dto.LabelPair{{Name: &n, Value: &v}}, Gauge: &dto.Gauge{Value: &gaugeValue}}
}

func strPtr(s string) *string { return &s }