fix(PROJ-86): Manticore-Index-Drift (emails_global) + Datumssortierung bei fehlendem Date-Header
emails_global bekam nie Tenant-Mails gespiegelt, Superadmin-Suche sah nur
~30% aller Mails. Zusätzlich sanken Mails ohne gültigen Date-Header
(date_ts=0) beim datumssortierten Listing ans Ende aller Ergebnisse und
waren dadurch praktisch unauffindbar ("GUI zeigt neue Mails nicht") -
Import/Indexierung liefen technisch korrekt, nur Sortierung/Spiegelung
waren kaputt.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UPFC6Jk2ke1Pq9XcuVGP1R
This commit is contained in:
sysops
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
7727ad5cdf
commit
26c0e04c75
@@ -8,6 +8,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archivmail/config"
|
||||
"archivmail/internal/index"
|
||||
@@ -129,7 +130,7 @@ func main() {
|
||||
Body: pm.TextBody + " " + pm.HTMLBody,
|
||||
AttachNames: strings.Join(attachNames, " "),
|
||||
HasAttachment: len(pm.Attachments) > 0,
|
||||
Date: pm.Date,
|
||||
Date: index.EffectiveDate(pm.Date, time.Now()),
|
||||
Size: int64(len(raw)),
|
||||
}
|
||||
|
||||
|
||||
@@ -243,6 +243,8 @@ func reindexOne(ctx context.Context, store *storage.Store, mgr index.TenantIndex
|
||||
}
|
||||
}
|
||||
|
||||
receivedAt, _ := store.GetReceivedAt(ctx, id)
|
||||
|
||||
doc := index.MailDocument{
|
||||
ID: id,
|
||||
From: pm.From,
|
||||
@@ -252,7 +254,7 @@ func reindexOne(ctx context.Context, store *storage.Store, mgr index.TenantIndex
|
||||
Body: pm.TextBody,
|
||||
AttachNames: strings.Join(attachNames, " "),
|
||||
HasAttachment: len(pm.Attachments) > 0,
|
||||
Date: pm.Date,
|
||||
Date: index.EffectiveDate(pm.Date, receivedAt),
|
||||
Size: int64(len(raw)),
|
||||
TenantID: tenantID,
|
||||
}
|
||||
|
||||
@@ -276,7 +276,7 @@ func importMessage(mailStore *storage.Store, idxMgr index.TenantIndexer, raw []b
|
||||
Body: pm.TextBody + " " + pm.HTMLBody,
|
||||
AttachNames: strings.Join(attachNames, " "),
|
||||
HasAttachment: len(pm.Attachments) > 0,
|
||||
Date: pm.Date,
|
||||
Date: index.EffectiveDate(pm.Date, time.Now()),
|
||||
Size: int64(len(raw)),
|
||||
TenantID: tenantID,
|
||||
}
|
||||
|
||||
@@ -139,7 +139,7 @@ func runIndexPending(args []string) {
|
||||
Body: pm.TextBody,
|
||||
AttachNames: strings.Join(attachNames, " "),
|
||||
HasAttachment: len(pm.Attachments) > 0,
|
||||
Date: pm.Date,
|
||||
Date: index.EffectiveDate(pm.Date, m.ReceivedAt),
|
||||
Size: int64(len(raw)),
|
||||
TenantID: m.TenantID,
|
||||
}
|
||||
|
||||
@@ -111,6 +111,11 @@ func runPurge(args []string) {
|
||||
if err := idxMgr.ForTenant(tenantID).Delete(id); err != nil {
|
||||
logger.Warn("purge: index cleanup failed", "id", id, "err", err)
|
||||
}
|
||||
if tenantID != nil {
|
||||
if err := idxMgr.Global().Delete(id); err != nil {
|
||||
logger.Warn("purge: global index cleanup failed", "id", id, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if audlog != nil {
|
||||
|
||||
@@ -107,6 +107,7 @@ func runReindex(args []string) {
|
||||
}
|
||||
|
||||
tenantID, _ := mailStore.GetTenantForMail(ctx, id)
|
||||
receivedAt, _ := mailStore.GetReceivedAt(ctx, id)
|
||||
|
||||
var attachNames []string
|
||||
for _, a := range pm.Attachments {
|
||||
@@ -124,7 +125,7 @@ func runReindex(args []string) {
|
||||
Body: pm.TextBody,
|
||||
AttachNames: strings.Join(attachNames, " "),
|
||||
HasAttachment: len(pm.Attachments) > 0,
|
||||
Date: pm.Date,
|
||||
Date: index.EffectiveDate(pm.Date, receivedAt),
|
||||
Size: int64(len(raw)),
|
||||
TenantID: tenantID,
|
||||
}
|
||||
@@ -148,6 +149,13 @@ func runReindex(args []string) {
|
||||
errors++
|
||||
continue
|
||||
}
|
||||
// Superadmin/global search reads emails_global — mirror every
|
||||
// tenant-scoped mail there too, not just untenanted ones.
|
||||
if tenantID != nil {
|
||||
if err := idxMgr.Global().IndexSync(doc); err != nil {
|
||||
logger.Warn("reindex: global index failed", "id", id, "err", err)
|
||||
}
|
||||
}
|
||||
indexed++
|
||||
|
||||
if (i+1)%100 == 0 {
|
||||
|
||||
@@ -433,7 +433,7 @@ func main() {
|
||||
smtpDaemon.SetIndexCallback(func(raw []byte, id string) {
|
||||
// Look up the tenant_id for this email from DB metadata.
|
||||
tenantID, _ := mailStore.GetTenantForMail(context.Background(), id)
|
||||
submitToWorker(tenantWorker, mailStore, raw, id, tenantID, logger, ocrWorker, cfg.Index.BatchMode, cfg.OCR.BatchMode)
|
||||
submitToWorker(tenantWorker, mailStore, raw, id, tenantID, logger, ocrWorker, cfg.Index.BatchMode, cfg.OCR.BatchMode, time.Now())
|
||||
})
|
||||
// Wire tenant routing into SMTP daemon
|
||||
if cfg.SMTP.TenantRouting == "domain" {
|
||||
@@ -592,7 +592,7 @@ func reloadOCRPauseWindow(configPath string, ocrWorker *ocr.Worker, logger *slog
|
||||
// in-memory submit is skipped so the unstarted batch-mode worker queue does
|
||||
// not fill up and log spurious "queue full" warnings. The mail still gets its
|
||||
// indexed_at / ocr_status state so the cron batch jobs pick it up.
|
||||
func submitToWorker(worker *index.TenantIndexWorker, store *storage.Store, raw []byte, id string, tenantID *int64, logger *slog.Logger, ocrWorker *ocr.Worker, indexBatchMode, ocrBatchMode bool) {
|
||||
func submitToWorker(worker *index.TenantIndexWorker, store *storage.Store, raw []byte, id string, tenantID *int64, logger *slog.Logger, ocrWorker *ocr.Worker, indexBatchMode, ocrBatchMode bool, dateFallback time.Time) {
|
||||
pm, err := mailparser.Parse(raw)
|
||||
if err != nil {
|
||||
logger.Warn("index: parse failed, skipping indexing", "id", id, "err", err)
|
||||
@@ -615,7 +615,7 @@ func submitToWorker(worker *index.TenantIndexWorker, store *storage.Store, raw [
|
||||
Body: pm.TextBody,
|
||||
AttachNames: strings.Join(attachNames, " "),
|
||||
HasAttachment: len(pm.Attachments) > 0,
|
||||
Date: pm.Date,
|
||||
Date: index.EffectiveDate(pm.Date, dateFallback),
|
||||
Size: int64(len(raw)),
|
||||
TenantID: tenantID,
|
||||
}
|
||||
@@ -688,9 +688,10 @@ func runBackfill(ctx context.Context, store *storage.Store, idx index.Indexer, w
|
||||
if !alreadyIndexed {
|
||||
needIndex++
|
||||
tenantID, _ := store.GetTenantForMail(ctx, id)
|
||||
receivedAt, _ := store.GetReceivedAt(ctx, id)
|
||||
// runBackfill only runs when index.batch_mode is off; the OCR
|
||||
// batch case is handled by passing a nil ocrWorker from the caller.
|
||||
submitToWorker(worker, store, raw, id, tenantID, logger, ocrWorker, false, false)
|
||||
submitToWorker(worker, store, raw, id, tenantID, logger, ocrWorker, false, false, receivedAt)
|
||||
}
|
||||
|
||||
if count%100 == 0 {
|
||||
@@ -731,6 +732,8 @@ func reindexTenant(ctx context.Context, store *storage.Store, mgr index.TenantIn
|
||||
continue
|
||||
}
|
||||
|
||||
receivedAt, _ := store.GetReceivedAt(ctx, id)
|
||||
|
||||
pm, parseErr := mailparser.Parse(raw)
|
||||
if parseErr != nil {
|
||||
logger.Warn("reindex tenant: parse failed", "tenant_id", tenantID, "id", id, "err", parseErr)
|
||||
@@ -754,7 +757,7 @@ func reindexTenant(ctx context.Context, store *storage.Store, mgr index.TenantIn
|
||||
Body: pm.TextBody,
|
||||
AttachNames: strings.Join(attachNames, " "),
|
||||
HasAttachment: len(pm.Attachments) > 0,
|
||||
Date: pm.Date,
|
||||
Date: index.EffectiveDate(pm.Date, receivedAt),
|
||||
Size: int64(len(raw)),
|
||||
TenantID: &tid,
|
||||
}
|
||||
|
||||
+2
-1
@@ -101,7 +101,8 @@
|
||||
| PROJ-83 | Audit-Logging für Anhang-Abrufe (GoBD/DSGVO-Nachbesserung) | Planned | [PROJ-83](PROJ-83-audit-log-anhang-abrufe.md) | 2026-08-06 |
|
||||
| PROJ-84 | Fix MIME-Header-Charset-Dekodierung + Backfill für Bestandsmails | Deployed | [PROJ-84](PROJ-84-fix-mime-header-charset-backfill.md) | 2026-08-06 |
|
||||
| PROJ-85 | Fix undeklarierte 8-Bit-Zeichen in Header/Body ohne Encoded-Word | Deployed | [PROJ-85](PROJ-85-fix-undeklarierte-8bit-header-charset.md) | 2026-08-06 |
|
||||
| PROJ-86 | Fix Manticore-Index-Drift (emails_global) + Datumssortierung bei fehlendem Date-Header | Deployed | [PROJ-86](PROJ-86-fix-manticore-index-drift-und-date-sortierung.md) | 2026-09-01 |
|
||||
|
||||
<!-- Add features above this line -->
|
||||
|
||||
## Next Available ID: PROJ-86
|
||||
## Next Available ID: PROJ-87
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
# PROJ-86: Fix Manticore-Index-Drift (emails_global) + Datumssortierung bei fehlendem Date-Header
|
||||
|
||||
## Status: Deployed
|
||||
**Created:** 2026-09-01
|
||||
**Last Updated:** 2026-09-01
|
||||
|
||||
## Kontext
|
||||
Ausgangspunkt: Nutzer meldete "Manticore-Index passt nicht" / "GUI zeigt aktuelle Mails aus dem Postfach nicht an". Diagnose auf 192.168.1.132 (192.168.1.131 gehört seit 2026-09-01 nicht mehr zum archivmail-Projekt) ergab zwei unabhängige Bugs:
|
||||
|
||||
1. **emails_global-Lücke:** Reindex und Live-Indexierung schrieben jede Mail nur in ihre Tenant-Tabelle (`ForTenant(tenantID)`), nie zusätzlich nach `emails_global`. Die Superadmin-Suche (`search_handlers.go`) liest bei `tenantID == nil` aber genau diesen globalen Index. Ergebnis: Superadmin sah nur ~30% aller Mails (16246 von 54202).
|
||||
2. **date_ts=0 bei fehlendem/kaputtem Date-Header:** Die Sortierung läuft standardmäßig über `date_ts DESC` (aus dem Mail-eigenen `Date:`-Header, `pm.Date`). Fehlt der Header oder ist er unparsebar (häufig bei Spam/Marketing-Mails), bleibt `pm.Date` Zero-Value → `date_ts=0` → die Mail sinkt beim datumssortierten Listing ans Ende aller Ergebnisse und taucht auf Seite 1 nie auf. Das war der eigentliche Grund für "GUI zeigt neue Mails nicht" — Import/Indexierung liefen technisch korrekt, die Mails waren nur unauffindbar einsortiert.
|
||||
|
||||
## Fix
|
||||
- Neue Hilfsfunktion `index.EffectiveDate(headerDate, fallback time.Time) time.Time` — nutzt `fallback` (received_at bei Reindex/Backfill, `time.Now()` bei Live-Ingest) wenn `headerDate` Zero-Value ist.
|
||||
- Neue Storage-Methode `Store.GetReceivedAt(ctx, id)`.
|
||||
- `Store.UnindexedMail` um Feld `ReceivedAt` erweitert (kein zusätzlicher DB-Roundtrip nötig).
|
||||
- An allen 9 Stellen, die `index.MailDocument{}` bauen, `Date: pm.Date` auf `Date: index.EffectiveDate(pm.Date, fallback)` umgestellt: `cmd_reindex.go`, `cmd_index_pending.go`, `cmd_fix_subjects.go`, `cmd_import.go`, `main.go` (submitToWorker + reindexTenant), `internal/imap/importer.go`, `internal/pop3/importer.go`, `internal/api/upload.go`, `cmd/archivmail-import/main.go`.
|
||||
- 5 Stellen um Spiegelung nach `emails_global` ergänzt (Write: `cmd_reindex.go`, `tenant_worker.go`, `internal/imap/importer.go`; Delete: `cmd_purge.go`, `internal/api/dsgvo_handlers.go`) — jeweils nur wenn `tenantID != nil` (sonst landet die Mail ohnehin direkt im globalen Index).
|
||||
|
||||
## Dependencies
|
||||
- Betrifft dieselbe Indexier-Infrastruktur wie PROJ-58 (Cron-Batch-Indexierung) und PROJ-65 (physische Tenant-Trennung) — die emails_global-Lücke entstand vermutlich mit PROJ-65.
|
||||
|
||||
## Acceptance Criteria
|
||||
- [x] `emails_global` enthält nach Reindex alle Tenant-Mails (54208 vs. DB-Gesamt 54202, Differenz = 6 Vorfix-Karteileichen, unkritisch)
|
||||
- [x] `date_ts=0`-Einträge nach Reindex in allen Tabellen (emails_global, emails_tenant_1/2/3) = 0
|
||||
- [x] Build + `go vet` grün auf 192.168.1.132
|
||||
- [x] Deployed auf 132 (einziger verbleibender Server), Voll-Reindex durchgeführt (54206/54206, 0 Fehler)
|
||||
- [x] Stichprobe verifiziert: Mail mit vormals `date_ts=0` hat nach Fix korrekten Timestamp (1788254163)
|
||||
|
||||
## Bekannte Restarbeit (nicht Teil dieses Tickets)
|
||||
- Vorfix-Karteileichen in `emails_global` (6 Stück, gelöschte Mails ohne Index-Cleanup vor diesem Fix) bereinigen sich beim nächsten regulären Purge-Zyklus von selbst — kein Handlungsbedarf.
|
||||
- 192.168.1.131 ist aus dem Projekt-Deployment-Ziel entfernt (Memory aktualisiert) — Skills/Doku, die noch 131 referenzieren, sollten bei Gelegenheit bereinigt werden.
|
||||
@@ -297,6 +297,9 @@ func (s *Server) handleDeleteDSGVOMails(w http.ResponseWriter, r *http.Request)
|
||||
continue
|
||||
}
|
||||
_ = searchIdx.Delete(m.MailID)
|
||||
if s.idxMgr != nil && tenantID != nil {
|
||||
_ = s.idxMgr.Global().Delete(m.MailID)
|
||||
}
|
||||
m.Deleted = true
|
||||
m.Deletable = false
|
||||
m.Reason = "Geloescht auf DSGVO-Antrag"
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"archivmail/internal/index"
|
||||
"archivmail/internal/safego"
|
||||
@@ -221,7 +222,7 @@ func (s *Server) importRawMessage(ctx context.Context, raw []byte, tenantID *int
|
||||
Body: pm.TextBody + " " + pm.HTMLBody,
|
||||
AttachNames: strings.Join(attachNames, " "),
|
||||
HasAttachment: len(pm.Attachments) > 0,
|
||||
Date: pm.Date,
|
||||
Date: index.EffectiveDate(pm.Date, time.Now()),
|
||||
Size: int64(len(raw)),
|
||||
}
|
||||
|
||||
|
||||
@@ -295,7 +295,7 @@ func (imp *Importer) storeAndIndex(raw []byte, tenantID *int64, accountID int64,
|
||||
Body: pm.TextBody,
|
||||
AttachNames: strings.Join(attachNames, " "),
|
||||
HasAttachment: len(pm.Attachments) > 0,
|
||||
Date: pm.Date,
|
||||
Date: index.EffectiveDate(pm.Date, time.Now()),
|
||||
Size: int64(len(raw)),
|
||||
TenantID: tenantID,
|
||||
}
|
||||
@@ -303,6 +303,12 @@ func (imp *Importer) storeAndIndex(raw []byte, tenantID *int64, accountID int64,
|
||||
if err := imp.idxMgr.ForTenant(tenantID).IndexSync(doc); err != nil {
|
||||
log.Warn("failed to index mail", "id", id, "err", err)
|
||||
// Non-fatal: mail is stored, just not searchable yet
|
||||
} else if tenantID != nil {
|
||||
// Superadmin/global search reads emails_global — mirror every
|
||||
// tenant-scoped mail there too, not just untenanted ones.
|
||||
if err := imp.idxMgr.Global().IndexSync(doc); err != nil {
|
||||
log.Warn("failed to index mail into global index", "id", id, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// PROJ-44: enqueue OCR job for any mail with attachments. Submit is
|
||||
|
||||
@@ -5,6 +5,18 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// EffectiveDate returns headerDate for sorting/filtering unless it is the
|
||||
// zero value (missing or unparseable Date: header — common in spam), in
|
||||
// which case fallback (typically received_at) is used instead. Without this,
|
||||
// mails with a bad Date header get date_ts=0 and sink to the bottom of every
|
||||
// date-sorted search result, effectively hiding them from the mail list.
|
||||
func EffectiveDate(headerDate, fallback time.Time) time.Time {
|
||||
if headerDate.IsZero() {
|
||||
return fallback
|
||||
}
|
||||
return headerDate
|
||||
}
|
||||
|
||||
// MailDocument is the indexed representation of a stored email.
|
||||
type MailDocument struct {
|
||||
ID string
|
||||
|
||||
@@ -100,5 +100,13 @@ func (w *TenantIndexWorker) indexDoc(doc MailDocument) {
|
||||
}
|
||||
if err := idx.IndexSync(doc); err != nil {
|
||||
w.logger.Error("tenant index worker: index failed", "id", doc.ID, "tenant_id", doc.TenantID, "err", err)
|
||||
return
|
||||
}
|
||||
// Superadmin/global search reads emails_global — mirror every
|
||||
// tenant-scoped mail there too, not just untenanted ones.
|
||||
if doc.TenantID != nil {
|
||||
if err := w.mgr.Global().IndexSync(doc); err != nil {
|
||||
w.logger.Error("tenant index worker: global index failed", "id", doc.ID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -173,7 +173,7 @@ func (imp *Importer) storeAndIndex(raw []byte, accountID int64, log *slog.Logger
|
||||
Body: pm.TextBody,
|
||||
AttachNames: strings.Join(attachNames, " "),
|
||||
HasAttachment: len(pm.Attachments) > 0,
|
||||
Date: pm.Date,
|
||||
Date: index.EffectiveDate(pm.Date, time.Now()),
|
||||
Size: int64(len(raw)),
|
||||
TenantID: imp.TenantID,
|
||||
}
|
||||
|
||||
@@ -1079,8 +1079,9 @@ func (s *Store) IsIndexed(ctx context.Context, id string) (bool, error) {
|
||||
// UnindexedMail describes one mail awaiting full-text indexing. TenantID is
|
||||
// nil when the mail has no tenant assignment (system-level / global).
|
||||
type UnindexedMail struct {
|
||||
ID string
|
||||
TenantID *int64
|
||||
ID string
|
||||
TenantID *int64
|
||||
ReceivedAt time.Time
|
||||
}
|
||||
|
||||
// GetUnindexedMails returns up to limit mails with indexed_at IS NULL, newest
|
||||
@@ -1090,7 +1091,7 @@ func (s *Store) GetUnindexedMails(ctx context.Context, limit int) ([]UnindexedMa
|
||||
if s.db == nil {
|
||||
return nil, nil
|
||||
}
|
||||
q := `SELECT id, tenant_id FROM emails WHERE indexed_at IS NULL ORDER BY received_at DESC`
|
||||
q := `SELECT id, tenant_id, received_at FROM emails WHERE indexed_at IS NULL ORDER BY received_at DESC`
|
||||
args := []interface{}{}
|
||||
if limit > 0 {
|
||||
q += " LIMIT $1"
|
||||
@@ -1104,7 +1105,7 @@ func (s *Store) GetUnindexedMails(ctx context.Context, limit int) ([]UnindexedMa
|
||||
var out []UnindexedMail
|
||||
for rows.Next() {
|
||||
var m UnindexedMail
|
||||
if err := rows.Scan(&m.ID, &m.TenantID); err != nil {
|
||||
if err := rows.Scan(&m.ID, &m.TenantID, &m.ReceivedAt); err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, m)
|
||||
@@ -1112,6 +1113,14 @@ func (s *Store) GetUnindexedMails(ctx context.Context, limit int) ([]UnindexedMa
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// GetReceivedAt returns the received_at timestamp for a single mail. Used as
|
||||
// the date-sort fallback when a mail's own Date: header is missing/unparseable.
|
||||
func (s *Store) GetReceivedAt(ctx context.Context, id string) (time.Time, error) {
|
||||
var t time.Time
|
||||
err := s.db.QueryRow(ctx, `SELECT received_at FROM emails WHERE id = $1`, id).Scan(&t)
|
||||
return t, err
|
||||
}
|
||||
|
||||
// ── Backfill ──────────────────────────────────────────────────────────────
|
||||
|
||||
// Backfill walks the store directory, parses each email, inserts missing DB
|
||||
|
||||
Reference in New Issue
Block a user