fix(PROJ-86): Manticore-Index-Drift (emails_global) + Datumssortierung bei fehlendem Date-Header

emails_global bekam nie Tenant-Mails gespiegelt, Superadmin-Suche sah nur
~30% aller Mails. Zusätzlich sanken Mails ohne gültigen Date-Header
(date_ts=0) beim datumssortierten Listing ans Ende aller Ergebnisse und
waren dadurch praktisch unauffindbar ("GUI zeigt neue Mails nicht") -
Import/Indexierung liefen technisch korrekt, nur Sortierung/Spiegelung
waren kaputt.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UPFC6Jk2ke1Pq9XcuVGP1R

cmd/archivmail-import/main.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"archivmail/config"
 	"archivmail/internal/index"
@@ -129,7 +130,7 @@ func main() {
 			Body:          pm.TextBody + " " + pm.HTMLBody,
 			AttachNames:   strings.Join(attachNames, " "),
 			HasAttachment: len(pm.Attachments) > 0,
-			Date:          pm.Date,
+			Date:          index.EffectiveDate(pm.Date, time.Now()),
 			Size:          int64(len(raw)),
 		}
 

cmd/archivmail/cmd_fix_subjects.go

@@ -243,6 +243,8 @@ func reindexOne(ctx context.Context, store *storage.Store, mgr index.TenantIndex
 		}
 	}
 
+	receivedAt, _ := store.GetReceivedAt(ctx, id)
+
 	doc := index.MailDocument{
 		ID:            id,
 		From:          pm.From,
@@ -252,7 +254,7 @@ func reindexOne(ctx context.Context, store *storage.Store, mgr index.TenantIndex
 		Body:          pm.TextBody,
 		AttachNames:   strings.Join(attachNames, " "),
 		HasAttachment: len(pm.Attachments) > 0,
-		Date:          pm.Date,
+		Date:          index.EffectiveDate(pm.Date, receivedAt),
 		Size:          int64(len(raw)),
 		TenantID:      tenantID,
 	}

cmd/archivmail/cmd_import.go

@@ -276,7 +276,7 @@ func importMessage(mailStore *storage.Store, idxMgr index.TenantIndexer, raw []b
 		Body:          pm.TextBody + " " + pm.HTMLBody,
 		AttachNames:   strings.Join(attachNames, " "),
 		HasAttachment: len(pm.Attachments) > 0,
-		Date:          pm.Date,
+		Date:          index.EffectiveDate(pm.Date, time.Now()),
 		Size:          int64(len(raw)),
 		TenantID:      tenantID,
 	}

cmd/archivmail/cmd_index_pending.go

@@ -139,7 +139,7 @@ func runIndexPending(args []string) {
 			Body:          pm.TextBody,
 			AttachNames:   strings.Join(attachNames, " "),
 			HasAttachment: len(pm.Attachments) > 0,
-			Date:          pm.Date,
+			Date:          index.EffectiveDate(pm.Date, m.ReceivedAt),
 			Size:          int64(len(raw)),
 			TenantID:      m.TenantID,
 		}

cmd/archivmail/cmd_purge.go

@@ -111,6 +111,11 @@ func runPurge(args []string) {
 			if err := idxMgr.ForTenant(tenantID).Delete(id); err != nil {
 				logger.Warn("purge: index cleanup failed", "id", id, "err", err)
 			}
+			if tenantID != nil {
+				if err := idxMgr.Global().Delete(id); err != nil {
+					logger.Warn("purge: global index cleanup failed", "id", id, "err", err)
+				}
+			}
 		}
 
 		if audlog != nil {

cmd/archivmail/cmd_reindex.go

@@ -107,6 +107,7 @@ func runReindex(args []string) {
 		}
 
 		tenantID, _ := mailStore.GetTenantForMail(ctx, id)
+		receivedAt, _ := mailStore.GetReceivedAt(ctx, id)
 
 		var attachNames []string
 		for _, a := range pm.Attachments {
@@ -124,7 +125,7 @@ func runReindex(args []string) {
 			Body:          pm.TextBody,
 			AttachNames:   strings.Join(attachNames, " "),
 			HasAttachment: len(pm.Attachments) > 0,
-			Date:          pm.Date,
+			Date:          index.EffectiveDate(pm.Date, receivedAt),
 			Size:          int64(len(raw)),
 			TenantID:      tenantID,
 		}
@@ -148,6 +149,13 @@ func runReindex(args []string) {
 			errors++
 			continue
 		}
+		// Superadmin/global search reads emails_global — mirror every
+		// tenant-scoped mail there too, not just untenanted ones.
+		if tenantID != nil {
+			if err := idxMgr.Global().IndexSync(doc); err != nil {
+				logger.Warn("reindex: global index failed", "id", id, "err", err)
+			}
+		}
 		indexed++
 
 		if (i+1)%100 == 0 {

cmd/archivmail/main.go

@@ -433,7 +433,7 @@ func main() {
 	smtpDaemon.SetIndexCallback(func(raw []byte, id string) {
 		// Look up the tenant_id for this email from DB metadata.
 		tenantID, _ := mailStore.GetTenantForMail(context.Background(), id)
-		submitToWorker(tenantWorker, mailStore, raw, id, tenantID, logger, ocrWorker, cfg.Index.BatchMode, cfg.OCR.BatchMode)
+		submitToWorker(tenantWorker, mailStore, raw, id, tenantID, logger, ocrWorker, cfg.Index.BatchMode, cfg.OCR.BatchMode, time.Now())
 	})
 	// Wire tenant routing into SMTP daemon
 	if cfg.SMTP.TenantRouting == "domain" {
@@ -592,7 +592,7 @@ func reloadOCRPauseWindow(configPath string, ocrWorker *ocr.Worker, logger *slog
 // in-memory submit is skipped so the unstarted batch-mode worker queue does
 // not fill up and log spurious "queue full" warnings. The mail still gets its
 // indexed_at / ocr_status state so the cron batch jobs pick it up.
-func submitToWorker(worker *index.TenantIndexWorker, store *storage.Store, raw []byte, id string, tenantID *int64, logger *slog.Logger, ocrWorker *ocr.Worker, indexBatchMode, ocrBatchMode bool) {
+func submitToWorker(worker *index.TenantIndexWorker, store *storage.Store, raw []byte, id string, tenantID *int64, logger *slog.Logger, ocrWorker *ocr.Worker, indexBatchMode, ocrBatchMode bool, dateFallback time.Time) {
 	pm, err := mailparser.Parse(raw)
 	if err != nil {
 		logger.Warn("index: parse failed, skipping indexing", "id", id, "err", err)
@@ -615,7 +615,7 @@ func submitToWorker(worker *index.TenantIndexWorker, store *storage.Store, raw [
 		Body:          pm.TextBody,
 		AttachNames:   strings.Join(attachNames, " "),
 		HasAttachment: len(pm.Attachments) > 0,
-		Date:          pm.Date,
+		Date:          index.EffectiveDate(pm.Date, dateFallback),
 		Size:          int64(len(raw)),
 		TenantID:      tenantID,
 	}
@@ -688,9 +688,10 @@ func runBackfill(ctx context.Context, store *storage.Store, idx index.Indexer, w
 		if !alreadyIndexed {
 			needIndex++
 			tenantID, _ := store.GetTenantForMail(ctx, id)
+			receivedAt, _ := store.GetReceivedAt(ctx, id)
 			// runBackfill only runs when index.batch_mode is off; the OCR
 			// batch case is handled by passing a nil ocrWorker from the caller.
-			submitToWorker(worker, store, raw, id, tenantID, logger, ocrWorker, false, false)
+			submitToWorker(worker, store, raw, id, tenantID, logger, ocrWorker, false, false, receivedAt)
 		}
 
 		if count%100 == 0 {
@@ -731,6 +732,8 @@ func reindexTenant(ctx context.Context, store *storage.Store, mgr index.TenantIn
 			continue
 		}
 
+		receivedAt, _ := store.GetReceivedAt(ctx, id)
+
 		pm, parseErr := mailparser.Parse(raw)
 		if parseErr != nil {
 			logger.Warn("reindex tenant: parse failed", "tenant_id", tenantID, "id", id, "err", parseErr)
@@ -754,7 +757,7 @@ func reindexTenant(ctx context.Context, store *storage.Store, mgr index.TenantIn
 			Body:          pm.TextBody,
 			AttachNames:   strings.Join(attachNames, " "),
 			HasAttachment: len(pm.Attachments) > 0,
-			Date:          pm.Date,
+			Date:          index.EffectiveDate(pm.Date, receivedAt),
 			Size:          int64(len(raw)),
 			TenantID:      &tid,
 		}

features/INDEX.md

@@ -101,7 +101,8 @@
 | PROJ-83 | Audit-Logging für Anhang-Abrufe (GoBD/DSGVO-Nachbesserung) | Planned | [PROJ-83](PROJ-83-audit-log-anhang-abrufe.md) | 2026-08-06 |
 | PROJ-84 | Fix MIME-Header-Charset-Dekodierung + Backfill für Bestandsmails | Deployed | [PROJ-84](PROJ-84-fix-mime-header-charset-backfill.md) | 2026-08-06 |
 | PROJ-85 | Fix undeklarierte 8-Bit-Zeichen in Header/Body ohne Encoded-Word | Deployed | [PROJ-85](PROJ-85-fix-undeklarierte-8bit-header-charset.md) | 2026-08-06 |
+| PROJ-86 | Fix Manticore-Index-Drift (emails_global) + Datumssortierung bei fehlendem Date-Header | Deployed | [PROJ-86](PROJ-86-fix-manticore-index-drift-und-date-sortierung.md) | 2026-09-01 |
 
 <!-- Add features above this line -->
 
-## Next Available ID: PROJ-86
+## Next Available ID: PROJ-87

features/PROJ-86-fix-manticore-index-drift-und-date-sortierung.md

@@ -0,0 +1,32 @@
+# PROJ-86: Fix Manticore-Index-Drift (emails_global) + Datumssortierung bei fehlendem Date-Header
+
+## Status: Deployed
+**Created:** 2026-09-01
+**Last Updated:** 2026-09-01
+
+## Kontext
+Ausgangspunkt: Nutzer meldete "Manticore-Index passt nicht" / "GUI zeigt aktuelle Mails aus dem Postfach nicht an". Diagnose auf 192.168.1.132 (192.168.1.131 gehört seit 2026-09-01 nicht mehr zum archivmail-Projekt) ergab zwei unabhängige Bugs:
+
+1. **emails_global-Lücke:** Reindex und Live-Indexierung schrieben jede Mail nur in ihre Tenant-Tabelle (`ForTenant(tenantID)`), nie zusätzlich nach `emails_global`. Die Superadmin-Suche (`search_handlers.go`) liest bei `tenantID == nil` aber genau diesen globalen Index. Ergebnis: Superadmin sah nur ~30% aller Mails (16246 von 54202).
+2. **date_ts=0 bei fehlendem/kaputtem Date-Header:** Die Sortierung läuft standardmäßig über `date_ts DESC` (aus dem Mail-eigenen `Date:`-Header, `pm.Date`). Fehlt der Header oder ist er unparsebar (häufig bei Spam/Marketing-Mails), bleibt `pm.Date` Zero-Value → `date_ts=0` → die Mail sinkt beim datumssortierten Listing ans Ende aller Ergebnisse und taucht auf Seite 1 nie auf. Das war der eigentliche Grund für "GUI zeigt neue Mails nicht" — Import/Indexierung liefen technisch korrekt, die Mails waren nur unauffindbar einsortiert.
+
+## Fix
+- Neue Hilfsfunktion `index.EffectiveDate(headerDate, fallback time.Time) time.Time` — nutzt `fallback` (received_at bei Reindex/Backfill, `time.Now()` bei Live-Ingest) wenn `headerDate` Zero-Value ist.
+- Neue Storage-Methode `Store.GetReceivedAt(ctx, id)`.
+- `Store.UnindexedMail` um Feld `ReceivedAt` erweitert (kein zusätzlicher DB-Roundtrip nötig).
+- An allen 9 Stellen, die `index.MailDocument{}` bauen, `Date: pm.Date` auf `Date: index.EffectiveDate(pm.Date, fallback)` umgestellt: `cmd_reindex.go`, `cmd_index_pending.go`, `cmd_fix_subjects.go`, `cmd_import.go`, `main.go` (submitToWorker + reindexTenant), `internal/imap/importer.go`, `internal/pop3/importer.go`, `internal/api/upload.go`, `cmd/archivmail-import/main.go`.
+- 5 Stellen um Spiegelung nach `emails_global` ergänzt (Write: `cmd_reindex.go`, `tenant_worker.go`, `internal/imap/importer.go`; Delete: `cmd_purge.go`, `internal/api/dsgvo_handlers.go`) — jeweils nur wenn `tenantID != nil` (sonst landet die Mail ohnehin direkt im globalen Index).
+
+## Dependencies
+- Betrifft dieselbe Indexier-Infrastruktur wie PROJ-58 (Cron-Batch-Indexierung) und PROJ-65 (physische Tenant-Trennung) — die emails_global-Lücke entstand vermutlich mit PROJ-65.
+
+## Acceptance Criteria
+- [x] `emails_global` enthält nach Reindex alle Tenant-Mails (54208 vs. DB-Gesamt 54202, Differenz = 6 Vorfix-Karteileichen, unkritisch)
+- [x] `date_ts=0`-Einträge nach Reindex in allen Tabellen (emails_global, emails_tenant_1/2/3) = 0
+- [x] Build + `go vet` grün auf 192.168.1.132
+- [x] Deployed auf 132 (einziger verbleibender Server), Voll-Reindex durchgeführt (54206/54206, 0 Fehler)
+- [x] Stichprobe verifiziert: Mail mit vormals `date_ts=0` hat nach Fix korrekten Timestamp (1788254163)
+
+## Bekannte Restarbeit (nicht Teil dieses Tickets)
+- Vorfix-Karteileichen in `emails_global` (6 Stück, gelöschte Mails ohne Index-Cleanup vor diesem Fix) bereinigen sich beim nächsten regulären Purge-Zyklus von selbst — kein Handlungsbedarf.
+- 192.168.1.131 ist aus dem Projekt-Deployment-Ziel entfernt (Memory aktualisiert) — Skills/Doku, die noch 131 referenzieren, sollten bei Gelegenheit bereinigt werden.

internal/api/dsgvo_handlers.go

@@ -297,6 +297,9 @@ func (s *Server) handleDeleteDSGVOMails(w http.ResponseWriter, r *http.Request)
 			continue
 		}
 		_ = searchIdx.Delete(m.MailID)
+		if s.idxMgr != nil && tenantID != nil {
+			_ = s.idxMgr.Global().Delete(m.MailID)
+		}
 		m.Deleted = true
 		m.Deletable = false
 		m.Reason = "Geloescht auf DSGVO-Antrag"

internal/api/upload.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"time"
 
 	"archivmail/internal/index"
 	"archivmail/internal/safego"
@@ -221,7 +222,7 @@ func (s *Server) importRawMessage(ctx context.Context, raw []byte, tenantID *int
 		Body:          pm.TextBody + " " + pm.HTMLBody,
 		AttachNames:   strings.Join(attachNames, " "),
 		HasAttachment: len(pm.Attachments) > 0,
-		Date:          pm.Date,
+		Date:          index.EffectiveDate(pm.Date, time.Now()),
 		Size:          int64(len(raw)),
 	}
 

internal/imap/importer.go

@@ -295,7 +295,7 @@ func (imp *Importer) storeAndIndex(raw []byte, tenantID *int64, accountID int64,
 		Body:          pm.TextBody,
 		AttachNames:   strings.Join(attachNames, " "),
 		HasAttachment: len(pm.Attachments) > 0,
-		Date:          pm.Date,
+		Date:          index.EffectiveDate(pm.Date, time.Now()),
 		Size:          int64(len(raw)),
 		TenantID:      tenantID,
 	}
@@ -303,6 +303,12 @@ func (imp *Importer) storeAndIndex(raw []byte, tenantID *int64, accountID int64,
 	if err := imp.idxMgr.ForTenant(tenantID).IndexSync(doc); err != nil {
 		log.Warn("failed to index mail", "id", id, "err", err)
 		// Non-fatal: mail is stored, just not searchable yet
+	} else if tenantID != nil {
+		// Superadmin/global search reads emails_global — mirror every
+		// tenant-scoped mail there too, not just untenanted ones.
+		if err := imp.idxMgr.Global().IndexSync(doc); err != nil {
+			log.Warn("failed to index mail into global index", "id", id, "err", err)
+		}
 	}
 
 	// PROJ-44: enqueue OCR job for any mail with attachments. Submit is

internal/index/index.go

@@ -5,6 +5,18 @@ import (
 	"time"
 )
 
+// EffectiveDate returns headerDate for sorting/filtering unless it is the
+// zero value (missing or unparseable Date: header — common in spam), in
+// which case fallback (typically received_at) is used instead. Without this,
+// mails with a bad Date header get date_ts=0 and sink to the bottom of every
+// date-sorted search result, effectively hiding them from the mail list.
+func EffectiveDate(headerDate, fallback time.Time) time.Time {
+	if headerDate.IsZero() {
+		return fallback
+	}
+	return headerDate
+}
+
 // MailDocument is the indexed representation of a stored email.
 type MailDocument struct {
 	ID             string

internal/index/tenant_worker.go

@@ -100,5 +100,13 @@ func (w *TenantIndexWorker) indexDoc(doc MailDocument) {
 	}
 	if err := idx.IndexSync(doc); err != nil {
 		w.logger.Error("tenant index worker: index failed", "id", doc.ID, "tenant_id", doc.TenantID, "err", err)
+		return
+	}
+	// Superadmin/global search reads emails_global — mirror every
+	// tenant-scoped mail there too, not just untenanted ones.
+	if doc.TenantID != nil {
+		if err := w.mgr.Global().IndexSync(doc); err != nil {
+			w.logger.Error("tenant index worker: global index failed", "id", doc.ID, "err", err)
+		}
 	}
 }

internal/pop3/importer.go

@@ -173,7 +173,7 @@ func (imp *Importer) storeAndIndex(raw []byte, accountID int64, log *slog.Logger
 		Body:          pm.TextBody,
 		AttachNames:   strings.Join(attachNames, " "),
 		HasAttachment: len(pm.Attachments) > 0,
-		Date:          pm.Date,
+		Date:          index.EffectiveDate(pm.Date, time.Now()),
 		Size:          int64(len(raw)),
 		TenantID:      imp.TenantID,
 	}

internal/storage/storage.go

@@ -1079,8 +1079,9 @@ func (s *Store) IsIndexed(ctx context.Context, id string) (bool, error) {
 // UnindexedMail describes one mail awaiting full-text indexing. TenantID is
 // nil when the mail has no tenant assignment (system-level / global).
 type UnindexedMail struct {
-	ID       string
+	ID         string
-	TenantID *int64
+	TenantID   *int64
+	ReceivedAt time.Time
 }
 
 // GetUnindexedMails returns up to limit mails with indexed_at IS NULL, newest
@@ -1090,7 +1091,7 @@ func (s *Store) GetUnindexedMails(ctx context.Context, limit int) ([]UnindexedMa
 	if s.db == nil {
 		return nil, nil
 	}
-	q := `SELECT id, tenant_id FROM emails WHERE indexed_at IS NULL ORDER BY received_at DESC`
+	q := `SELECT id, tenant_id, received_at FROM emails WHERE indexed_at IS NULL ORDER BY received_at DESC`
 	args := []interface{}{}
 	if limit > 0 {
 		q += " LIMIT $1"
@@ -1104,7 +1105,7 @@ func (s *Store) GetUnindexedMails(ctx context.Context, limit int) ([]UnindexedMa
 	var out []UnindexedMail
 	for rows.Next() {
 		var m UnindexedMail
-		if err := rows.Scan(&m.ID, &m.TenantID); err != nil {
+		if err := rows.Scan(&m.ID, &m.TenantID, &m.ReceivedAt); err != nil {
 			continue
 		}
 		out = append(out, m)
@@ -1112,6 +1113,14 @@ func (s *Store) GetUnindexedMails(ctx context.Context, limit int) ([]UnindexedMa
 	return out, rows.Err()
 }
 
+// GetReceivedAt returns the received_at timestamp for a single mail. Used as
+// the date-sort fallback when a mail's own Date: header is missing/unparseable.
+func (s *Store) GetReceivedAt(ctx context.Context, id string) (time.Time, error) {
+	var t time.Time
+	err := s.db.QueryRow(ctx, `SELECT received_at FROM emails WHERE id = $1`, id).Scan(&t)
+	return t, err
+}
+
 // ── Backfill ──────────────────────────────────────────────────────────────
 
 // Backfill walks the store directory, parses each email, inserts missing DB