feat(PROJ-84): fix-subjects CLI-Backfill für kaputte MIME-Header-Betreffs

CLI-Subkommando "archivmail fix-subjects" korrigiert Bestandsmails mit undekodiertem RFC-2047-Encoded-Word im Betreff (Folge des Fixes in 31d7113). Dry-Run per Default, echte Änderung nur mit --apply. Original-EML im Storage bleibt unangetastet - nur Postgres emails.subject wird korrigiert, danach Manticore-Reindex inkl. Erhalt von OCR-Text. Jeder Lauf erzeugt einen Audit-Log-Eintrag (metadata_backfill). Verifiziert auf 192.168.1.132: 543 Kandidaten, 541 reparierbar, 2 zu Recht übersprungen (RFC-2047-Verletzung im Original). --apply noch nicht ausgeführt. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WapWkrQusDuBMhaN8WyuXB
2026-08-06 19:58:41 +02:00
parent 31d71138c9
commit 1dfd8c18ac
9 changed files with 500 additions and 1 deletions
@@ -0,0 +1,319 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log/slog"
+	"os"
+	"strings"
+
+	"archivmail/config"
+	"archivmail/internal/audit"
+	"archivmail/internal/index"
+	"archivmail/internal/storage"
+	"archivmail/pkg/mailparser"
+)
+
+// runFixSubjects repairs `emails.subject` rows that still contain a raw,
+// undecoded RFC 2047 encoded-word (e.g. `=?Windows-1252?Q?...?=`). Those were
+// written before decodeMIMEHeader() got a CharsetReader for charsets outside
+// UTF-8/US-ASCII/ISO-8859-1.
+//
+// Scope of the repair:
+//   - ONLY the PostgreSQL metadata column `subject` is rewritten. The archived
+//     original (encrypted EML in the store) is never touched — this is a
+//     display/search metadata repair, not a change to the immutable archive
+//     copy (same principle as PROJ-34 / PROJ-48).
+//   - After a successful DB update the mail is re-indexed in Manticore
+//     (emails_global / emails_tenant_N), otherwise search would keep showing
+//     the old subject.
+//   - The run itself is written to the audit log (system action).
+//
+// Dry-run is the DEFAULT: without --apply nothing is written.
+//
+// Usage:
+//
+//	archivmail fix-subjects                    # dry-run over all tenants
+//	archivmail fix-subjects --tenant 2 --limit 50
+//	archivmail fix-subjects --apply
+func runFixSubjects(args []string) {
+	fs := flag.NewFlagSet("fix-subjects", flag.ExitOnError)
+	configPath := fs.String("config", "/etc/archivmail/config.yml", "path to config file")
+	tenantIDFlag := fs.Int64("tenant", 0, "tenant ID (0 = all tenants)")
+	limitFlag := fs.Int("limit", 0, "max number of mails to inspect (0 = no limit)")
+	applyFlag := fs.Bool("apply", false, "actually write changes (default: dry-run only)")
+	verboseFlag := fs.Bool("verbose", false, "log every affected subject (old -> new)")
+	fs.Parse(args)
+
+	logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
+
+	cfg, err := config.Load(*configPath)
+	if err != nil {
+		logger.Error("failed to load config", "err", err)
+		os.Exit(1)
+	}
+
+	storeCfg := storage.Config{
+		Dir:             cfg.Storage.StorePath,
+		Keyfile:         cfg.Storage.Keyfile,
+		DSN:             cfg.Database.DSN(),
+		CompressEnabled: cfg.Storage.Compress,
+	}
+	mailStore, err := storage.New(storeCfg)
+	if err != nil {
+		logger.Error("storage init failed", "err", err)
+		os.Exit(1)
+	}
+	defer mailStore.Close()
+
+	ctx := context.Background()
+
+	var tenantPtr *int64
+	if *tenantIDFlag > 0 {
+		t := *tenantIDFlag
+		tenantPtr = &t
+	}
+
+	// Index manager is only needed when we actually write.
+	var idxMgr index.TenantIndexer
+	if *applyFlag {
+		idxMgr, err = openIndexManager(cfg)
+		if err != nil {
+			logger.Error("index init failed", "err", err)
+			os.Exit(1)
+		}
+		defer func() { idxMgr.Close() }()
+	}
+
+	rows, err := mailStore.ListRawEncodedSubjects(ctx, tenantPtr, *limitFlag)
+	if err != nil {
+		logger.Error("failed to list candidate subjects", "err", err)
+		os.Exit(1)
+	}
+
+	mode := "DRY-RUN"
+	if *applyFlag {
+		mode = "APPLY"
+	}
+	logger.Info("fix-subjects: starting",
+		"mode", mode, "tenant", *tenantIDFlag, "candidates", len(rows))
+
+	var (
+		affected     int // subject really contains an encoded-word and decodes differently
+		updated      int
+		undecodable  int // still encoded after decoding attempt
+		reindexed    int
+		indexSkipped int
+		errCount     int
+	)
+
+	for _, row := range rows {
+		if !mailparser.HasEncodedWord(row.Subject) {
+			continue
+		}
+
+		// Preferred source of truth: re-parse the archived original (read-only).
+		// Falls back to decoding the stored string if the mail cannot be read.
+		newSubject := ""
+		var pm *mailparser.ParsedMail
+		var raw []byte
+		if data, lerr := mailStore.Load(row.ID); lerr == nil {
+			if parsed, perr := mailparser.Parse(data); perr == nil {
+				pm = parsed
+				raw = data
+				newSubject = parsed.Subject
+			} else {
+				logger.Warn("fix-subjects: parse failed, falling back to header decode",
+					"id", row.ID, "err", perr)
+			}
+		} else {
+			logger.Warn("fix-subjects: load failed, falling back to header decode",
+				"id", row.ID, "err", lerr)
+		}
+		if newSubject == "" {
+			newSubject = mailparser.DecodeMIMEHeader(row.Subject)
+		}
+
+		if newSubject == row.Subject {
+			undecodable++
+			logger.Warn("fix-subjects: subject still not decodable", "id", row.ID, "subject", row.Subject)
+			continue
+		}
+		affected++
+
+		if *verboseFlag || !*applyFlag {
+			logger.Info("fix-subjects: would fix",
+				"id", row.ID, "tenant", tenantLabel(row.TenantID),
+				"old", row.Subject, "new", newSubject)
+		}
+		if !*applyFlag {
+			continue
+		}
+
+		if err := mailStore.UpdateSubjectMetadata(ctx, row.ID, newSubject); err != nil {
+			logger.Warn("fix-subjects: update failed", "id", row.ID, "err", err)
+			errCount++
+			continue
+		}
+		updated++
+
+		// Pull the corrected subject into the full-text index.
+		if pm == nil {
+			indexSkipped++
+			logger.Warn("fix-subjects: mail not parseable — index not updated", "id", row.ID)
+			continue
+		}
+		if err := reindexOne(ctx, mailStore, idxMgr, row.ID, row.TenantID, pm, raw); err != nil {
+			logger.Warn("fix-subjects: reindex failed", "id", row.ID, "err", err)
+			errCount++
+			continue
+		}
+		reindexed++
+	}
+
+	logger.Info("fix-subjects: complete",
+		"mode", mode,
+		"candidates", len(rows),
+		"affected", affected,
+		"updated", updated,
+		"reindexed", reindexed,
+		"index_skipped", indexSkipped,
+		"undecodable", undecodable,
+		"errors", errCount)
+
+	writeFixSubjectsAudit(cfg, logger, fixSubjectsAudit{
+		Apply:        *applyFlag,
+		TenantID:     tenantPtr,
+		Candidates:   len(rows),
+		Affected:     affected,
+		Updated:      updated,
+		Reindexed:    reindexed,
+		IndexSkipped: indexSkipped,
+		Undecodable:  undecodable,
+		Errors:       errCount,
+	})
+
+	if errCount > 0 {
+		os.Exit(1)
+	}
+}
+
+// openIndexManager builds the configured index backend (manticore by default).
+func openIndexManager(cfg *config.Config) (index.TenantIndexer, error) {
+	backend := cfg.Index.Backend
+	if backend == "" {
+		backend = "manticore"
+	}
+	if backend == "manticore" {
+		dsn := cfg.Index.ManticoreDSN
+		if dsn == "" {
+			dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4"
+		}
+		m, err := index.NewManticoreTenantManager(dsn)
+		if err != nil {
+			return nil, fmt.Errorf("manticore init: %w", err)
+		}
+		return m, nil
+	}
+	batchSize := cfg.Index.BatchSize
+	if batchSize <= 0 {
+		batchSize = 100
+	}
+	m, err := index.NewTenantIndexManager(cfg.Index.Path, batchSize, backend)
+	if err != nil {
+		return nil, fmt.Errorf("index manager init: %w", err)
+	}
+	return m, nil
+}
+
+// reindexOne re-indexes a single mail from an already parsed message. Existing
+// OCR text is preserved because IndexSync uses REPLACE (see runReindex).
+func reindexOne(ctx context.Context, store *storage.Store, mgr index.TenantIndexer,
+	id string, tenantID *int64, pm *mailparser.ParsedMail, raw []byte) error {
+
+	var attachNames []string
+	for _, a := range pm.Attachments {
+		if a.Filename != "" {
+			attachNames = append(attachNames, a.Filename)
+		}
+	}
+
+	doc := index.MailDocument{
+		ID:            id,
+		From:          pm.From,
+		To:            strings.Join(pm.To, ", "),
+		CC:            strings.Join(pm.CC, ", "),
+		Subject:       pm.Subject,
+		Body:          pm.TextBody,
+		AttachNames:   strings.Join(attachNames, " "),
+		HasAttachment: len(pm.Attachments) > 0,
+		Date:          pm.Date,
+		Size:          int64(len(raw)),
+		TenantID:      tenantID,
+	}
+
+	idx := mgr.ForTenant(tenantID)
+
+	if _, ocrChars, err := store.GetOCRMeta(ctx, id); err == nil && ocrChars > 0 {
+		if reader, ok := idx.(index.AttachmentTextReader); ok {
+			if existing, rerr := reader.GetAttachmentText(id); rerr == nil && existing != "" {
+				doc.AttachmentText = existing
+			}
+		}
+	}
+
+	if err := idx.IndexSync(doc); err != nil {
+		return fmt.Errorf("index sync: %w", err)
+	}
+	return nil
+}
+
+// fixSubjectsAudit carries the counters of one backfill run into the audit log.
+type fixSubjectsAudit struct {
+	Apply        bool
+	TenantID     *int64
+	Candidates   int
+	Affected     int
+	Updated      int
+	Reindexed    int
+	IndexSkipped int
+	Undecodable  int
+	Errors       int
+}
+
+// writeFixSubjectsAudit records the run (dry-run included) in the audit log.
+// Audit failures must never abort the maintenance run itself.
+func writeFixSubjectsAudit(cfg *config.Config, logger *slog.Logger, a fixSubjectsAudit) {
+	audlog, err := audit.New(cfg.Database.DSN(), cfg.Audit.ResolvedLogPath(), logger)
+	if err != nil {
+		logger.Warn("fix-subjects: audit log unavailable", "err", err)
+		return
+	}
+	defer audlog.Close()
+
+	mode := "dry-run"
+	if a.Apply {
+		mode = "apply"
+	}
+	audlog.Log(audit.Entry{
+		EventType: audit.EventMetadataBackfill,
+		Username:  "system",
+		IPAddress: "cli",
+		Query:     "fix-subjects",
+		Success:   a.Errors == 0,
+		TenantID:  a.TenantID,
+		Detail: fmt.Sprintf(
+			"fix-subjects (%s): candidates=%d affected=%d updated=%d reindexed=%d index_skipped=%d undecodable=%d errors=%d; "+
+				"nur emails.subject korrigiert, Archivoriginal unverändert",
+			mode, a.Candidates, a.Affected, a.Updated, a.Reindexed, a.IndexSkipped, a.Undecodable, a.Errors),
+	})
+}
+
+// tenantLabel renders a nullable tenant ID for log output.
+func tenantLabel(t *int64) string {
+	if t == nil {
+		return "global"
+	}
+	return fmt.Sprintf("%d", *t)
+}
@@ -308,6 +308,7 @@ Commands:
  purge          Mails mit abgelaufener Aufbewahrungsfrist löschen (cron-fähig)
  recompress     Bestehende Mails nachträglich gzip-komprimieren
  rethread       Thread-IDs rückwirkend aus In-Reply-To/References befüllen
+  fix-subjects   Kaputte RFC-2047-Betreffs in den Metadaten reparieren (Dry-Run per Default)
  ocr-reprocess  OCR für Anhänge nachholen (alle oder pro Mandant/Status)
  index-pending  Ungeindexte Mails nachindexieren (cron-fähig, PROJ-58 batch_mode)
  backup         Store, Keyfile, PostgreSQL und Config konsistent sichern (PROJ-66)
@@ -78,6 +78,9 @@ func main() {
 		case "rethread":
 			runRethread(os.Args[2:])
 			return
+		case "fix-subjects":
+			runFixSubjects(os.Args[2:])
+			return
 		case "ocr-reprocess":
 			runOCRReprocess(os.Args[2:])
 			return