feat(PROJ-52): Vollständigkeits-Reconciliation (Zähl-Report Mailserver vs. Archiv)
Täglicher Cron-Job (archivmail reconcile) berechnet pro Tenant/Quelle (SMTP-Journal, IMAP-Konto, POP3-Konto, Datei-Import) archivierte Mail-Zahlen, für IMAP zusätzlich einen Soll/Ist-Vergleich via UID-Tracking. Abweichungen über Schwellenwert erzeugen Audit-Log-Warnung. Neue Admin-Dashboard-Kachel "Vollständigkeits-Check" (letzte 7 Tage, Warn-Badge, CSV-Export). Schließt die "teilweise erfüllt"-Lücke bei Vollständigkeit im GoBD/DSGVO-Compliance-Check (VOI-Grundsatz 2). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
b286352d07
commit
be93614c9f
@@ -253,6 +253,11 @@ func importMessage(mailStore *storage.Store, idxMgr index.TenantIndexer, raw []b
|
||||
return "error"
|
||||
}
|
||||
|
||||
// PROJ-52: mark bulk-imported mails as source 'import' for reconciliation.
|
||||
if err := mailStore.TagSource(context.Background(), id, "import", nil); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "warning: tag source failed: %v\n", err)
|
||||
}
|
||||
|
||||
var attachNames []string
|
||||
for _, a := range pm.Attachments {
|
||||
attachNames = append(attachNames, a.Filename)
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"archivmail/config"
|
||||
"archivmail/internal/audit"
|
||||
"archivmail/internal/reconciliation"
|
||||
)
|
||||
|
||||
// runReconcile computes the daily completeness reconciliation report (PROJ-52).
|
||||
// It is designed to be driven by cron once per day (e.g. shortly after
|
||||
// midnight) and, by default, reconciles the *previous* full calendar day so a
|
||||
// day is only counted once it is complete.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// archivmail reconcile --config /etc/archivmail/config.yml
|
||||
// archivmail reconcile --date 2026-07-01
|
||||
// archivmail reconcile --days 7 # backfill: reconcile the last 7 days
|
||||
func runReconcile(args []string) {
|
||||
fs := flag.NewFlagSet("reconcile", flag.ExitOnError)
|
||||
configPath := fs.String("config", "/etc/archivmail/config.yml", "path to config file")
|
||||
dateFlag := fs.String("date", "", "day to reconcile (YYYY-MM-DD, UTC); default: yesterday")
|
||||
daysFlag := fs.Int("days", 1, "number of days to reconcile, ending at --date (backfill)")
|
||||
fs.Parse(args)
|
||||
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
|
||||
cfg, err := config.Load(*configPath)
|
||||
if err != nil {
|
||||
logger.Error("failed to load config", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Determine the target (end) date.
|
||||
var end time.Time
|
||||
if *dateFlag != "" {
|
||||
end, err = time.ParseInLocation("2006-01-02", *dateFlag, time.UTC)
|
||||
if err != nil {
|
||||
logger.Error("invalid --date (expected YYYY-MM-DD)", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
} else {
|
||||
end = time.Now().UTC().AddDate(0, 0, -1).Truncate(24 * time.Hour)
|
||||
}
|
||||
|
||||
days := *daysFlag
|
||||
if days < 1 {
|
||||
days = 1
|
||||
}
|
||||
|
||||
dsn := cfg.Database.DSN()
|
||||
reconStore, err := reconciliation.New(dsn, logger)
|
||||
if err != nil {
|
||||
logger.Error("reconciliation store init failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer reconStore.Close()
|
||||
|
||||
// Wire audit logging so anomalies are persisted as tenant-visible entries.
|
||||
audlog, err := audit.New(dsn, cfg.Audit.ResolvedLogPath(), logger)
|
||||
if err != nil {
|
||||
logger.Warn("audit init failed — anomalies will only be logged", "err", err)
|
||||
} else {
|
||||
defer audlog.Close()
|
||||
reconStore.SetAuditLogger(audlog)
|
||||
}
|
||||
|
||||
thresholdPct := cfg.Reconciliation.ResolvedThresholdPct()
|
||||
ctx := context.Background()
|
||||
|
||||
totalAnomalies := 0
|
||||
// Reconcile oldest → newest so trailing-average history is populated in order.
|
||||
for i := days - 1; i >= 0; i-- {
|
||||
day := end.AddDate(0, 0, -i)
|
||||
anomalies, err := reconStore.ComputeForDate(ctx, day, thresholdPct)
|
||||
if err != nil {
|
||||
// Per the spec: on failure the day is left WITHOUT a report row
|
||||
// (no false zeros). Exit non-zero so cron surfaces the failure.
|
||||
logger.Error("reconcile: compute failed", "date", day.Format("2006-01-02"), "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
totalAnomalies += len(anomalies)
|
||||
logger.Info("reconcile: day complete",
|
||||
"date", day.Format("2006-01-02"), "anomalies", len(anomalies))
|
||||
}
|
||||
|
||||
logger.Info("reconcile: complete", "days", days, "threshold_pct", thresholdPct,
|
||||
"anomalies_total", totalAnomalies)
|
||||
}
|
||||
+18
-1
@@ -31,6 +31,7 @@ import (
|
||||
"archivmail/internal/mailer"
|
||||
"archivmail/internal/ocr"
|
||||
pop3store "archivmail/internal/pop3"
|
||||
"archivmail/internal/reconciliation"
|
||||
"archivmail/internal/smtpoutconfig"
|
||||
"archivmail/internal/smtpd"
|
||||
"archivmail/internal/storage"
|
||||
@@ -73,6 +74,9 @@ func main() {
|
||||
case "index-pending":
|
||||
runIndexPending(os.Args[2:])
|
||||
return
|
||||
case "reconcile":
|
||||
runReconcile(os.Args[2:])
|
||||
return
|
||||
case "update":
|
||||
runUpdate(os.Args[2:])
|
||||
return
|
||||
@@ -313,11 +317,24 @@ func main() {
|
||||
srv.SetGlobalRetentionDays(cfg.Storage.RetentionDays)
|
||||
srv.SetMetrics(cfg.Metrics)
|
||||
|
||||
// PROJ-52: completeness reconciliation store — powers the dashboard +
|
||||
// CSV-export endpoints. The daily computation itself is driven by cron via
|
||||
// the `archivmail reconcile` subcommand (analog PROJ-58 batch jobs), so the
|
||||
// daemon only needs read access here.
|
||||
reconStore, err := reconciliation.New(cfg.Database.DSN(), logger)
|
||||
if err != nil {
|
||||
logger.Error("reconciliation store init failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer reconStore.Close()
|
||||
reconStore.SetAuditLogger(audlog)
|
||||
srv.SetReconciliation(reconStore, cfg.Reconciliation.ResolvedThresholdPct())
|
||||
|
||||
// PROJ-28: Self-Service Onboarding — mailer + token store + FQDN
|
||||
mlr := mailer.New(cfg.SMTPOut)
|
||||
|
||||
// SMTP-Out config store — load from DB, overrides config.yml if present
|
||||
smtpOutSt, err := smtpoutconfig.New(cfg.Database.DSN(), cfg.API.Secret)
|
||||
smtpOutSt, err := smtpoutconfig.New(cfg.Database.DSN(), aesKey)
|
||||
if err != nil {
|
||||
logger.Error("smtp-out config store init failed", "err", err)
|
||||
os.Exit(1)
|
||||
|
||||
Reference in New Issue
Block a user