From 26c0e04c7530e1706bad43132e4a25405f99b393 Mon Sep 17 00:00:00 2001 From: sysops Date: Tue, 1 Sep 2026 12:17:09 +0200 Subject: [PATCH] fix(PROJ-86): Manticore-Index-Drift (emails_global) + Datumssortierung bei fehlendem Date-Header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit emails_global bekam nie Tenant-Mails gespiegelt, Superadmin-Suche sah nur ~30% aller Mails. Zusätzlich sanken Mails ohne gültigen Date-Header (date_ts=0) beim datumssortierten Listing ans Ende aller Ergebnisse und waren dadurch praktisch unauffindbar ("GUI zeigt neue Mails nicht") - Import/Indexierung liefen technisch korrekt, nur Sortierung/Spiegelung waren kaputt. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01UPFC6Jk2ke1Pq9XcuVGP1R --- cmd/archivmail-import/main.go | 3 +- cmd/archivmail/cmd_fix_subjects.go | 4 ++- cmd/archivmail/cmd_import.go | 2 +- cmd/archivmail/cmd_index_pending.go | 2 +- cmd/archivmail/cmd_purge.go | 5 +++ cmd/archivmail/cmd_reindex.go | 10 +++++- cmd/archivmail/main.go | 13 +++++--- features/INDEX.md | 3 +- ...nticore-index-drift-und-date-sortierung.md | 32 +++++++++++++++++++ internal/api/dsgvo_handlers.go | 3 ++ internal/api/upload.go | 3 +- internal/imap/importer.go | 8 ++++- internal/index/index.go | 12 +++++++ internal/index/tenant_worker.go | 8 +++++ internal/pop3/importer.go | 2 +- internal/storage/storage.go | 17 +++++++--- 16 files changed, 109 insertions(+), 18 deletions(-) create mode 100644 features/PROJ-86-fix-manticore-index-drift-und-date-sortierung.md diff --git a/cmd/archivmail-import/main.go b/cmd/archivmail-import/main.go index c23b8e7..9a109a7 100644 --- a/cmd/archivmail-import/main.go +++ b/cmd/archivmail-import/main.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "time" "archivmail/config" "archivmail/internal/index" @@ -129,7 +130,7 @@ func main() { Body: pm.TextBody + " " + pm.HTMLBody, AttachNames: strings.Join(attachNames, " "), HasAttachment: len(pm.Attachments) > 0, - Date: pm.Date, + Date: index.EffectiveDate(pm.Date, time.Now()), Size: int64(len(raw)), } diff --git a/cmd/archivmail/cmd_fix_subjects.go b/cmd/archivmail/cmd_fix_subjects.go index 362947f..049fa29 100644 --- a/cmd/archivmail/cmd_fix_subjects.go +++ b/cmd/archivmail/cmd_fix_subjects.go @@ -243,6 +243,8 @@ func reindexOne(ctx context.Context, store *storage.Store, mgr index.TenantIndex } } + receivedAt, _ := store.GetReceivedAt(ctx, id) + doc := index.MailDocument{ ID: id, From: pm.From, @@ -252,7 +254,7 @@ func reindexOne(ctx context.Context, store *storage.Store, mgr index.TenantIndex Body: pm.TextBody, AttachNames: strings.Join(attachNames, " "), HasAttachment: len(pm.Attachments) > 0, - Date: pm.Date, + Date: index.EffectiveDate(pm.Date, receivedAt), Size: int64(len(raw)), TenantID: tenantID, } diff --git a/cmd/archivmail/cmd_import.go b/cmd/archivmail/cmd_import.go index 4c8c684..33b7dac 100644 --- a/cmd/archivmail/cmd_import.go +++ b/cmd/archivmail/cmd_import.go @@ -276,7 +276,7 @@ func importMessage(mailStore *storage.Store, idxMgr index.TenantIndexer, raw []b Body: pm.TextBody + " " + pm.HTMLBody, AttachNames: strings.Join(attachNames, " "), HasAttachment: len(pm.Attachments) > 0, - Date: pm.Date, + Date: index.EffectiveDate(pm.Date, time.Now()), Size: int64(len(raw)), TenantID: tenantID, } diff --git a/cmd/archivmail/cmd_index_pending.go b/cmd/archivmail/cmd_index_pending.go index c083beb..a66b1ec 100644 --- a/cmd/archivmail/cmd_index_pending.go +++ b/cmd/archivmail/cmd_index_pending.go @@ -139,7 +139,7 @@ func runIndexPending(args []string) { Body: pm.TextBody, AttachNames: strings.Join(attachNames, " "), HasAttachment: len(pm.Attachments) > 0, - Date: pm.Date, + Date: index.EffectiveDate(pm.Date, m.ReceivedAt), Size: int64(len(raw)), TenantID: m.TenantID, } diff --git a/cmd/archivmail/cmd_purge.go b/cmd/archivmail/cmd_purge.go index 2388179..0cfd06a 100644 --- a/cmd/archivmail/cmd_purge.go +++ b/cmd/archivmail/cmd_purge.go @@ -111,6 +111,11 @@ func runPurge(args []string) { if err := idxMgr.ForTenant(tenantID).Delete(id); err != nil { logger.Warn("purge: index cleanup failed", "id", id, "err", err) } + if tenantID != nil { + if err := idxMgr.Global().Delete(id); err != nil { + logger.Warn("purge: global index cleanup failed", "id", id, "err", err) + } + } } if audlog != nil { diff --git a/cmd/archivmail/cmd_reindex.go b/cmd/archivmail/cmd_reindex.go index 7df6152..2f0315c 100644 --- a/cmd/archivmail/cmd_reindex.go +++ b/cmd/archivmail/cmd_reindex.go @@ -107,6 +107,7 @@ func runReindex(args []string) { } tenantID, _ := mailStore.GetTenantForMail(ctx, id) + receivedAt, _ := mailStore.GetReceivedAt(ctx, id) var attachNames []string for _, a := range pm.Attachments { @@ -124,7 +125,7 @@ func runReindex(args []string) { Body: pm.TextBody, AttachNames: strings.Join(attachNames, " "), HasAttachment: len(pm.Attachments) > 0, - Date: pm.Date, + Date: index.EffectiveDate(pm.Date, receivedAt), Size: int64(len(raw)), TenantID: tenantID, } @@ -148,6 +149,13 @@ func runReindex(args []string) { errors++ continue } + // Superadmin/global search reads emails_global — mirror every + // tenant-scoped mail there too, not just untenanted ones. + if tenantID != nil { + if err := idxMgr.Global().IndexSync(doc); err != nil { + logger.Warn("reindex: global index failed", "id", id, "err", err) + } + } indexed++ if (i+1)%100 == 0 { diff --git a/cmd/archivmail/main.go b/cmd/archivmail/main.go index 351e4e9..0c23a32 100644 --- a/cmd/archivmail/main.go +++ b/cmd/archivmail/main.go @@ -433,7 +433,7 @@ func main() { smtpDaemon.SetIndexCallback(func(raw []byte, id string) { // Look up the tenant_id for this email from DB metadata. tenantID, _ := mailStore.GetTenantForMail(context.Background(), id) - submitToWorker(tenantWorker, mailStore, raw, id, tenantID, logger, ocrWorker, cfg.Index.BatchMode, cfg.OCR.BatchMode) + submitToWorker(tenantWorker, mailStore, raw, id, tenantID, logger, ocrWorker, cfg.Index.BatchMode, cfg.OCR.BatchMode, time.Now()) }) // Wire tenant routing into SMTP daemon if cfg.SMTP.TenantRouting == "domain" { @@ -592,7 +592,7 @@ func reloadOCRPauseWindow(configPath string, ocrWorker *ocr.Worker, logger *slog // in-memory submit is skipped so the unstarted batch-mode worker queue does // not fill up and log spurious "queue full" warnings. The mail still gets its // indexed_at / ocr_status state so the cron batch jobs pick it up. -func submitToWorker(worker *index.TenantIndexWorker, store *storage.Store, raw []byte, id string, tenantID *int64, logger *slog.Logger, ocrWorker *ocr.Worker, indexBatchMode, ocrBatchMode bool) { +func submitToWorker(worker *index.TenantIndexWorker, store *storage.Store, raw []byte, id string, tenantID *int64, logger *slog.Logger, ocrWorker *ocr.Worker, indexBatchMode, ocrBatchMode bool, dateFallback time.Time) { pm, err := mailparser.Parse(raw) if err != nil { logger.Warn("index: parse failed, skipping indexing", "id", id, "err", err) @@ -615,7 +615,7 @@ func submitToWorker(worker *index.TenantIndexWorker, store *storage.Store, raw [ Body: pm.TextBody, AttachNames: strings.Join(attachNames, " "), HasAttachment: len(pm.Attachments) > 0, - Date: pm.Date, + Date: index.EffectiveDate(pm.Date, dateFallback), Size: int64(len(raw)), TenantID: tenantID, } @@ -688,9 +688,10 @@ func runBackfill(ctx context.Context, store *storage.Store, idx index.Indexer, w if !alreadyIndexed { needIndex++ tenantID, _ := store.GetTenantForMail(ctx, id) + receivedAt, _ := store.GetReceivedAt(ctx, id) // runBackfill only runs when index.batch_mode is off; the OCR // batch case is handled by passing a nil ocrWorker from the caller. - submitToWorker(worker, store, raw, id, tenantID, logger, ocrWorker, false, false) + submitToWorker(worker, store, raw, id, tenantID, logger, ocrWorker, false, false, receivedAt) } if count%100 == 0 { @@ -731,6 +732,8 @@ func reindexTenant(ctx context.Context, store *storage.Store, mgr index.TenantIn continue } + receivedAt, _ := store.GetReceivedAt(ctx, id) + pm, parseErr := mailparser.Parse(raw) if parseErr != nil { logger.Warn("reindex tenant: parse failed", "tenant_id", tenantID, "id", id, "err", parseErr) @@ -754,7 +757,7 @@ func reindexTenant(ctx context.Context, store *storage.Store, mgr index.TenantIn Body: pm.TextBody, AttachNames: strings.Join(attachNames, " "), HasAttachment: len(pm.Attachments) > 0, - Date: pm.Date, + Date: index.EffectiveDate(pm.Date, receivedAt), Size: int64(len(raw)), TenantID: &tid, } diff --git a/features/INDEX.md b/features/INDEX.md index 7b3a295..e01b9f8 100644 --- a/features/INDEX.md +++ b/features/INDEX.md @@ -101,7 +101,8 @@ | PROJ-83 | Audit-Logging für Anhang-Abrufe (GoBD/DSGVO-Nachbesserung) | Planned | [PROJ-83](PROJ-83-audit-log-anhang-abrufe.md) | 2026-08-06 | | PROJ-84 | Fix MIME-Header-Charset-Dekodierung + Backfill für Bestandsmails | Deployed | [PROJ-84](PROJ-84-fix-mime-header-charset-backfill.md) | 2026-08-06 | | PROJ-85 | Fix undeklarierte 8-Bit-Zeichen in Header/Body ohne Encoded-Word | Deployed | [PROJ-85](PROJ-85-fix-undeklarierte-8bit-header-charset.md) | 2026-08-06 | +| PROJ-86 | Fix Manticore-Index-Drift (emails_global) + Datumssortierung bei fehlendem Date-Header | Deployed | [PROJ-86](PROJ-86-fix-manticore-index-drift-und-date-sortierung.md) | 2026-09-01 | -## Next Available ID: PROJ-86 +## Next Available ID: PROJ-87 diff --git a/features/PROJ-86-fix-manticore-index-drift-und-date-sortierung.md b/features/PROJ-86-fix-manticore-index-drift-und-date-sortierung.md new file mode 100644 index 0000000..b26144c --- /dev/null +++ b/features/PROJ-86-fix-manticore-index-drift-und-date-sortierung.md @@ -0,0 +1,32 @@ +# PROJ-86: Fix Manticore-Index-Drift (emails_global) + Datumssortierung bei fehlendem Date-Header + +## Status: Deployed +**Created:** 2026-09-01 +**Last Updated:** 2026-09-01 + +## Kontext +Ausgangspunkt: Nutzer meldete "Manticore-Index passt nicht" / "GUI zeigt aktuelle Mails aus dem Postfach nicht an". Diagnose auf 192.168.1.132 (192.168.1.131 gehört seit 2026-09-01 nicht mehr zum archivmail-Projekt) ergab zwei unabhängige Bugs: + +1. **emails_global-Lücke:** Reindex und Live-Indexierung schrieben jede Mail nur in ihre Tenant-Tabelle (`ForTenant(tenantID)`), nie zusätzlich nach `emails_global`. Die Superadmin-Suche (`search_handlers.go`) liest bei `tenantID == nil` aber genau diesen globalen Index. Ergebnis: Superadmin sah nur ~30% aller Mails (16246 von 54202). +2. **date_ts=0 bei fehlendem/kaputtem Date-Header:** Die Sortierung läuft standardmäßig über `date_ts DESC` (aus dem Mail-eigenen `Date:`-Header, `pm.Date`). Fehlt der Header oder ist er unparsebar (häufig bei Spam/Marketing-Mails), bleibt `pm.Date` Zero-Value → `date_ts=0` → die Mail sinkt beim datumssortierten Listing ans Ende aller Ergebnisse und taucht auf Seite 1 nie auf. Das war der eigentliche Grund für "GUI zeigt neue Mails nicht" — Import/Indexierung liefen technisch korrekt, die Mails waren nur unauffindbar einsortiert. + +## Fix +- Neue Hilfsfunktion `index.EffectiveDate(headerDate, fallback time.Time) time.Time` — nutzt `fallback` (received_at bei Reindex/Backfill, `time.Now()` bei Live-Ingest) wenn `headerDate` Zero-Value ist. +- Neue Storage-Methode `Store.GetReceivedAt(ctx, id)`. +- `Store.UnindexedMail` um Feld `ReceivedAt` erweitert (kein zusätzlicher DB-Roundtrip nötig). +- An allen 9 Stellen, die `index.MailDocument{}` bauen, `Date: pm.Date` auf `Date: index.EffectiveDate(pm.Date, fallback)` umgestellt: `cmd_reindex.go`, `cmd_index_pending.go`, `cmd_fix_subjects.go`, `cmd_import.go`, `main.go` (submitToWorker + reindexTenant), `internal/imap/importer.go`, `internal/pop3/importer.go`, `internal/api/upload.go`, `cmd/archivmail-import/main.go`. +- 5 Stellen um Spiegelung nach `emails_global` ergänzt (Write: `cmd_reindex.go`, `tenant_worker.go`, `internal/imap/importer.go`; Delete: `cmd_purge.go`, `internal/api/dsgvo_handlers.go`) — jeweils nur wenn `tenantID != nil` (sonst landet die Mail ohnehin direkt im globalen Index). + +## Dependencies +- Betrifft dieselbe Indexier-Infrastruktur wie PROJ-58 (Cron-Batch-Indexierung) und PROJ-65 (physische Tenant-Trennung) — die emails_global-Lücke entstand vermutlich mit PROJ-65. + +## Acceptance Criteria +- [x] `emails_global` enthält nach Reindex alle Tenant-Mails (54208 vs. DB-Gesamt 54202, Differenz = 6 Vorfix-Karteileichen, unkritisch) +- [x] `date_ts=0`-Einträge nach Reindex in allen Tabellen (emails_global, emails_tenant_1/2/3) = 0 +- [x] Build + `go vet` grün auf 192.168.1.132 +- [x] Deployed auf 132 (einziger verbleibender Server), Voll-Reindex durchgeführt (54206/54206, 0 Fehler) +- [x] Stichprobe verifiziert: Mail mit vormals `date_ts=0` hat nach Fix korrekten Timestamp (1788254163) + +## Bekannte Restarbeit (nicht Teil dieses Tickets) +- Vorfix-Karteileichen in `emails_global` (6 Stück, gelöschte Mails ohne Index-Cleanup vor diesem Fix) bereinigen sich beim nächsten regulären Purge-Zyklus von selbst — kein Handlungsbedarf. +- 192.168.1.131 ist aus dem Projekt-Deployment-Ziel entfernt (Memory aktualisiert) — Skills/Doku, die noch 131 referenzieren, sollten bei Gelegenheit bereinigt werden. diff --git a/internal/api/dsgvo_handlers.go b/internal/api/dsgvo_handlers.go index fb25ed8..a946e8d 100644 --- a/internal/api/dsgvo_handlers.go +++ b/internal/api/dsgvo_handlers.go @@ -297,6 +297,9 @@ func (s *Server) handleDeleteDSGVOMails(w http.ResponseWriter, r *http.Request) continue } _ = searchIdx.Delete(m.MailID) + if s.idxMgr != nil && tenantID != nil { + _ = s.idxMgr.Global().Delete(m.MailID) + } m.Deleted = true m.Deletable = false m.Reason = "Geloescht auf DSGVO-Antrag" diff --git a/internal/api/upload.go b/internal/api/upload.go index 150a502..043b816 100644 --- a/internal/api/upload.go +++ b/internal/api/upload.go @@ -7,6 +7,7 @@ import ( "net/http" "strings" "sync" + "time" "archivmail/internal/index" "archivmail/internal/safego" @@ -221,7 +222,7 @@ func (s *Server) importRawMessage(ctx context.Context, raw []byte, tenantID *int Body: pm.TextBody + " " + pm.HTMLBody, AttachNames: strings.Join(attachNames, " "), HasAttachment: len(pm.Attachments) > 0, - Date: pm.Date, + Date: index.EffectiveDate(pm.Date, time.Now()), Size: int64(len(raw)), } diff --git a/internal/imap/importer.go b/internal/imap/importer.go index 9f6b7e6..6a1265e 100644 --- a/internal/imap/importer.go +++ b/internal/imap/importer.go @@ -295,7 +295,7 @@ func (imp *Importer) storeAndIndex(raw []byte, tenantID *int64, accountID int64, Body: pm.TextBody, AttachNames: strings.Join(attachNames, " "), HasAttachment: len(pm.Attachments) > 0, - Date: pm.Date, + Date: index.EffectiveDate(pm.Date, time.Now()), Size: int64(len(raw)), TenantID: tenantID, } @@ -303,6 +303,12 @@ func (imp *Importer) storeAndIndex(raw []byte, tenantID *int64, accountID int64, if err := imp.idxMgr.ForTenant(tenantID).IndexSync(doc); err != nil { log.Warn("failed to index mail", "id", id, "err", err) // Non-fatal: mail is stored, just not searchable yet + } else if tenantID != nil { + // Superadmin/global search reads emails_global — mirror every + // tenant-scoped mail there too, not just untenanted ones. + if err := imp.idxMgr.Global().IndexSync(doc); err != nil { + log.Warn("failed to index mail into global index", "id", id, "err", err) + } } // PROJ-44: enqueue OCR job for any mail with attachments. Submit is diff --git a/internal/index/index.go b/internal/index/index.go index cfa6315..cb90335 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -5,6 +5,18 @@ import ( "time" ) +// EffectiveDate returns headerDate for sorting/filtering unless it is the +// zero value (missing or unparseable Date: header — common in spam), in +// which case fallback (typically received_at) is used instead. Without this, +// mails with a bad Date header get date_ts=0 and sink to the bottom of every +// date-sorted search result, effectively hiding them from the mail list. +func EffectiveDate(headerDate, fallback time.Time) time.Time { + if headerDate.IsZero() { + return fallback + } + return headerDate +} + // MailDocument is the indexed representation of a stored email. type MailDocument struct { ID string diff --git a/internal/index/tenant_worker.go b/internal/index/tenant_worker.go index f9a5047..a087c22 100644 --- a/internal/index/tenant_worker.go +++ b/internal/index/tenant_worker.go @@ -100,5 +100,13 @@ func (w *TenantIndexWorker) indexDoc(doc MailDocument) { } if err := idx.IndexSync(doc); err != nil { w.logger.Error("tenant index worker: index failed", "id", doc.ID, "tenant_id", doc.TenantID, "err", err) + return + } + // Superadmin/global search reads emails_global — mirror every + // tenant-scoped mail there too, not just untenanted ones. + if doc.TenantID != nil { + if err := w.mgr.Global().IndexSync(doc); err != nil { + w.logger.Error("tenant index worker: global index failed", "id", doc.ID, "err", err) + } } } diff --git a/internal/pop3/importer.go b/internal/pop3/importer.go index 70153e1..1b1a082 100644 --- a/internal/pop3/importer.go +++ b/internal/pop3/importer.go @@ -173,7 +173,7 @@ func (imp *Importer) storeAndIndex(raw []byte, accountID int64, log *slog.Logger Body: pm.TextBody, AttachNames: strings.Join(attachNames, " "), HasAttachment: len(pm.Attachments) > 0, - Date: pm.Date, + Date: index.EffectiveDate(pm.Date, time.Now()), Size: int64(len(raw)), TenantID: imp.TenantID, } diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 2cdfe59..83e0121 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -1079,8 +1079,9 @@ func (s *Store) IsIndexed(ctx context.Context, id string) (bool, error) { // UnindexedMail describes one mail awaiting full-text indexing. TenantID is // nil when the mail has no tenant assignment (system-level / global). type UnindexedMail struct { - ID string - TenantID *int64 + ID string + TenantID *int64 + ReceivedAt time.Time } // GetUnindexedMails returns up to limit mails with indexed_at IS NULL, newest @@ -1090,7 +1091,7 @@ func (s *Store) GetUnindexedMails(ctx context.Context, limit int) ([]UnindexedMa if s.db == nil { return nil, nil } - q := `SELECT id, tenant_id FROM emails WHERE indexed_at IS NULL ORDER BY received_at DESC` + q := `SELECT id, tenant_id, received_at FROM emails WHERE indexed_at IS NULL ORDER BY received_at DESC` args := []interface{}{} if limit > 0 { q += " LIMIT $1" @@ -1104,7 +1105,7 @@ func (s *Store) GetUnindexedMails(ctx context.Context, limit int) ([]UnindexedMa var out []UnindexedMail for rows.Next() { var m UnindexedMail - if err := rows.Scan(&m.ID, &m.TenantID); err != nil { + if err := rows.Scan(&m.ID, &m.TenantID, &m.ReceivedAt); err != nil { continue } out = append(out, m) @@ -1112,6 +1113,14 @@ func (s *Store) GetUnindexedMails(ctx context.Context, limit int) ([]UnindexedMa return out, rows.Err() } +// GetReceivedAt returns the received_at timestamp for a single mail. Used as +// the date-sort fallback when a mail's own Date: header is missing/unparseable. +func (s *Store) GetReceivedAt(ctx context.Context, id string) (time.Time, error) { + var t time.Time + err := s.db.QueryRow(ctx, `SELECT received_at FROM emails WHERE id = $1`, id).Scan(&t) + return t, err +} + // ── Backfill ────────────────────────────────────────────────────────────── // Backfill walks the store directory, parses each email, inserts missing DB