feat(PROJ-52): Vollständigkeits-Reconciliation (Zähl-Report Mailserver vs. Archiv)

Täglicher Cron-Job (archivmail reconcile) berechnet pro Tenant/Quelle
(SMTP-Journal, IMAP-Konto, POP3-Konto, Datei-Import) archivierte Mail-Zahlen,
für IMAP zusätzlich einen Soll/Ist-Vergleich via UID-Tracking. Abweichungen
über Schwellenwert erzeugen Audit-Log-Warnung. Neue Admin-Dashboard-Kachel
"Vollständigkeits-Check" (letzte 7 Tage, Warn-Badge, CSV-Export).

Schließt die "teilweise erfüllt"-Lücke bei Vollständigkeit im
GoBD/DSGVO-Compliance-Check (VOI-Grundsatz 2).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
sysops
2026-07-03 22:48:42 +02:00
co-authored by Claude Sonnet 5
parent b286352d07
commit be93614c9f
24 changed files with 1577 additions and 10 deletions
+5
View File
@@ -253,6 +253,11 @@ func importMessage(mailStore *storage.Store, idxMgr index.TenantIndexer, raw []b
return "error"
}
// PROJ-52: mark bulk-imported mails as source 'import' for reconciliation.
if err := mailStore.TagSource(context.Background(), id, "import", nil); err != nil {
fmt.Fprintf(os.Stderr, "warning: tag source failed: %v\n", err)
}
var attachNames []string
for _, a := range pm.Attachments {
attachNames = append(attachNames, a.Filename)
+95
View File
@@ -0,0 +1,95 @@
package main
import (
"context"
"flag"
"log/slog"
"os"
"time"
"archivmail/config"
"archivmail/internal/audit"
"archivmail/internal/reconciliation"
)
// runReconcile computes the daily completeness reconciliation report (PROJ-52).
// It is designed to be driven by cron once per day (e.g. shortly after
// midnight) and, by default, reconciles the *previous* full calendar day so a
// day is only counted once it is complete.
//
// Usage:
//
// archivmail reconcile --config /etc/archivmail/config.yml
// archivmail reconcile --date 2026-07-01
// archivmail reconcile --days 7 # backfill: reconcile the last 7 days
func runReconcile(args []string) {
fs := flag.NewFlagSet("reconcile", flag.ExitOnError)
configPath := fs.String("config", "/etc/archivmail/config.yml", "path to config file")
dateFlag := fs.String("date", "", "day to reconcile (YYYY-MM-DD, UTC); default: yesterday")
daysFlag := fs.Int("days", 1, "number of days to reconcile, ending at --date (backfill)")
fs.Parse(args)
logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
cfg, err := config.Load(*configPath)
if err != nil {
logger.Error("failed to load config", "err", err)
os.Exit(1)
}
// Determine the target (end) date.
var end time.Time
if *dateFlag != "" {
end, err = time.ParseInLocation("2006-01-02", *dateFlag, time.UTC)
if err != nil {
logger.Error("invalid --date (expected YYYY-MM-DD)", "err", err)
os.Exit(1)
}
} else {
end = time.Now().UTC().AddDate(0, 0, -1).Truncate(24 * time.Hour)
}
days := *daysFlag
if days < 1 {
days = 1
}
dsn := cfg.Database.DSN()
reconStore, err := reconciliation.New(dsn, logger)
if err != nil {
logger.Error("reconciliation store init failed", "err", err)
os.Exit(1)
}
defer reconStore.Close()
// Wire audit logging so anomalies are persisted as tenant-visible entries.
audlog, err := audit.New(dsn, cfg.Audit.ResolvedLogPath(), logger)
if err != nil {
logger.Warn("audit init failed — anomalies will only be logged", "err", err)
} else {
defer audlog.Close()
reconStore.SetAuditLogger(audlog)
}
thresholdPct := cfg.Reconciliation.ResolvedThresholdPct()
ctx := context.Background()
totalAnomalies := 0
// Reconcile oldest → newest so trailing-average history is populated in order.
for i := days - 1; i >= 0; i-- {
day := end.AddDate(0, 0, -i)
anomalies, err := reconStore.ComputeForDate(ctx, day, thresholdPct)
if err != nil {
// Per the spec: on failure the day is left WITHOUT a report row
// (no false zeros). Exit non-zero so cron surfaces the failure.
logger.Error("reconcile: compute failed", "date", day.Format("2006-01-02"), "err", err)
os.Exit(1)
}
totalAnomalies += len(anomalies)
logger.Info("reconcile: day complete",
"date", day.Format("2006-01-02"), "anomalies", len(anomalies))
}
logger.Info("reconcile: complete", "days", days, "threshold_pct", thresholdPct,
"anomalies_total", totalAnomalies)
}
+18 -1
View File
@@ -31,6 +31,7 @@ import (
"archivmail/internal/mailer"
"archivmail/internal/ocr"
pop3store "archivmail/internal/pop3"
"archivmail/internal/reconciliation"
"archivmail/internal/smtpoutconfig"
"archivmail/internal/smtpd"
"archivmail/internal/storage"
@@ -73,6 +74,9 @@ func main() {
case "index-pending":
runIndexPending(os.Args[2:])
return
case "reconcile":
runReconcile(os.Args[2:])
return
case "update":
runUpdate(os.Args[2:])
return
@@ -313,11 +317,24 @@ func main() {
srv.SetGlobalRetentionDays(cfg.Storage.RetentionDays)
srv.SetMetrics(cfg.Metrics)
// PROJ-52: completeness reconciliation store — powers the dashboard +
// CSV-export endpoints. The daily computation itself is driven by cron via
// the `archivmail reconcile` subcommand (analog PROJ-58 batch jobs), so the
// daemon only needs read access here.
reconStore, err := reconciliation.New(cfg.Database.DSN(), logger)
if err != nil {
logger.Error("reconciliation store init failed", "err", err)
os.Exit(1)
}
defer reconStore.Close()
reconStore.SetAuditLogger(audlog)
srv.SetReconciliation(reconStore, cfg.Reconciliation.ResolvedThresholdPct())
// PROJ-28: Self-Service Onboarding — mailer + token store + FQDN
mlr := mailer.New(cfg.SMTPOut)
// SMTP-Out config store — load from DB, overrides config.yml if present
smtpOutSt, err := smtpoutconfig.New(cfg.Database.DSN(), cfg.API.Secret)
smtpOutSt, err := smtpoutconfig.New(cfg.Database.DSN(), aesKey)
if err != nil {
logger.Error("smtp-out config store init failed", "err", err)
os.Exit(1)