scripte/archivmail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(PROJ-86): Manticore-Index-Drift (emails_global) + Datumssortierung bei fehlendem Date-Header

Browse Source 
emails_global bekam nie Tenant-Mails gespiegelt, Superadmin-Suche sah nur
~30% aller Mails. Zusätzlich sanken Mails ohne gültigen Date-Header
(date_ts=0) beim datumssortierten Listing ans Ende aller Ergebnisse und
waren dadurch praktisch unauffindbar ("GUI zeigt neue Mails nicht") -
Import/Indexierung liefen technisch korrekt, nur Sortierung/Spiegelung
waren kaputt.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UPFC6Jk2ke1Pq9XcuVGP1R
This commit is contained in:
sysops
2026-09-01 12:17:09 +02:00
co-authored by Claude Sonnet 5
parent 7727ad5cdf
commit 26c0e04c75
16 changed files with 109 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/archivmail-import/main.go
+2 -1
View File
@@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"strings"
"time"
"archivmail/config"
"archivmail/internal/index"
@@ -129,7 +130,7 @@ func main() {
Body: pm.TextBody + " " + pm.HTMLBody,
AttachNames: strings.Join(attachNames, " "),
HasAttachment: len(pm.Attachments) > 0,
Date: pm.Date,
Date: index.EffectiveDate(pm.Date, time.Now()),
Size: int64(len(raw)),
}
cmd/archivmail/cmd_fix_subjects.go
+3 -1
View File
@@ -243,6 +243,8 @@ func reindexOne(ctx context.Context, store *storage.Store, mgr index.TenantIndex
}
}
receivedAt, _ := store.GetReceivedAt(ctx, id)
doc := index.MailDocument{
ID: id,
From: pm.From,
@@ -252,7 +254,7 @@ func reindexOne(ctx context.Context, store *storage.Store, mgr index.TenantIndex
Body: pm.TextBody,
AttachNames: strings.Join(attachNames, " "),
HasAttachment: len(pm.Attachments) > 0,
Date: pm.Date,
Date: index.EffectiveDate(pm.Date, receivedAt),
Size: int64(len(raw)),
TenantID: tenantID,
}
cmd/archivmail/cmd_import.go
+1 -1
View File
@@ -276,7 +276,7 @@ func importMessage(mailStore *storage.Store, idxMgr index.TenantIndexer, raw []b
Body: pm.TextBody + " " + pm.HTMLBody,
AttachNames: strings.Join(attachNames, " "),
HasAttachment: len(pm.Attachments) > 0,
Date: pm.Date,
Date: index.EffectiveDate(pm.Date, time.Now()),
Size: int64(len(raw)),
TenantID: tenantID,
}
cmd/archivmail/cmd_index_pending.go
+1 -1
View File
@@ -139,7 +139,7 @@ func runIndexPending(args []string) {
Body: pm.TextBody,
AttachNames: strings.Join(attachNames, " "),
HasAttachment: len(pm.Attachments) > 0,
Date: pm.Date,
Date: index.EffectiveDate(pm.Date, m.ReceivedAt),
Size: int64(len(raw)),
TenantID: m.TenantID,
}
cmd/archivmail/cmd_purge.go
+5
View File
@@ -111,6 +111,11 @@ func runPurge(args []string) {
if err := idxMgr.ForTenant(tenantID).Delete(id); err != nil {
logger.Warn("purge: index cleanup failed", "id", id, "err", err)
}
if tenantID != nil {
if err := idxMgr.Global().Delete(id); err != nil {
logger.Warn("purge: global index cleanup failed", "id", id, "err", err)
}
}
}
if audlog != nil {
cmd/archivmail/cmd_reindex.go
+9 -1
View File
@@ -107,6 +107,7 @@ func runReindex(args []string) {
}
tenantID, _ := mailStore.GetTenantForMail(ctx, id)
receivedAt, _ := mailStore.GetReceivedAt(ctx, id)
var attachNames []string
for _, a := range pm.Attachments {
@@ -124,7 +125,7 @@ func runReindex(args []string) {
Body: pm.TextBody,
AttachNames: strings.Join(attachNames, " "),
HasAttachment: len(pm.Attachments) > 0,
Date: pm.Date,
Date: index.EffectiveDate(pm.Date, receivedAt),
Size: int64(len(raw)),
TenantID: tenantID,
}
@@ -148,6 +149,13 @@ func runReindex(args []string) {
errors++
continue
}
// Superadmin/global search reads emails_global — mirror every
// tenant-scoped mail there too, not just untenanted ones.
if tenantID != nil {
if err := idxMgr.Global().IndexSync(doc); err != nil {
logger.Warn("reindex: global index failed", "id", id, "err", err)
}
}
indexed++
if (i+1)%100 == 0 {
cmd/archivmail/main.go
+8 -5
View File
@@ -433,7 +433,7 @@ func main() {
smtpDaemon.SetIndexCallback(func(raw []byte, id string) {
// Look up the tenant_id for this email from DB metadata.
tenantID, _ := mailStore.GetTenantForMail(context.Background(), id)
submitToWorker(tenantWorker, mailStore, raw, id, tenantID, logger, ocrWorker, cfg.Index.BatchMode, cfg.OCR.BatchMode)
submitToWorker(tenantWorker, mailStore, raw, id, tenantID, logger, ocrWorker, cfg.Index.BatchMode, cfg.OCR.BatchMode, time.Now())
})
// Wire tenant routing into SMTP daemon
if cfg.SMTP.TenantRouting == "domain" {
@@ -592,7 +592,7 @@ func reloadOCRPauseWindow(configPath string, ocrWorker *ocr.Worker, logger *slog
// in-memory submit is skipped so the unstarted batch-mode worker queue does
// not fill up and log spurious "queue full" warnings. The mail still gets its
// indexed_at / ocr_status state so the cron batch jobs pick it up.
func submitToWorker(worker *index.TenantIndexWorker, store *storage.Store, raw []byte, id string, tenantID *int64, logger *slog.Logger, ocrWorker *ocr.Worker, indexBatchMode, ocrBatchMode bool) {
func submitToWorker(worker *index.TenantIndexWorker, store *storage.Store, raw []byte, id string, tenantID *int64, logger *slog.Logger, ocrWorker *ocr.Worker, indexBatchMode, ocrBatchMode bool, dateFallback time.Time) {
pm, err := mailparser.Parse(raw)
if err != nil {
logger.Warn("index: parse failed, skipping indexing", "id", id, "err", err)
@@ -615,7 +615,7 @@ func submitToWorker(worker *index.TenantIndexWorker, store *storage.Store, raw [
Body: pm.TextBody,
AttachNames: strings.Join(attachNames, " "),
HasAttachment: len(pm.Attachments) > 0,
Date: pm.Date,
Date: index.EffectiveDate(pm.Date, dateFallback),
Size: int64(len(raw)),
TenantID: tenantID,
}
@@ -688,9 +688,10 @@ func runBackfill(ctx context.Context, store *storage.Store, idx index.Indexer, w
if !alreadyIndexed {
needIndex++
tenantID, _ := store.GetTenantForMail(ctx, id)
receivedAt, _ := store.GetReceivedAt(ctx, id)
// runBackfill only runs when index.batch_mode is off; the OCR
// batch case is handled by passing a nil ocrWorker from the caller.
submitToWorker(worker, store, raw, id, tenantID, logger, ocrWorker, false, false)
submitToWorker(worker, store, raw, id, tenantID, logger, ocrWorker, false, false, receivedAt)
}
if count%100 == 0 {
@@ -731,6 +732,8 @@ func reindexTenant(ctx context.Context, store *storage.Store, mgr index.TenantIn
continue
}
receivedAt, _ := store.GetReceivedAt(ctx, id)
pm, parseErr := mailparser.Parse(raw)
if parseErr != nil {
logger.Warn("reindex tenant: parse failed", "tenant_id", tenantID, "id", id, "err", parseErr)
@@ -754,7 +757,7 @@ func reindexTenant(ctx context.Context, store *storage.Store, mgr index.TenantIn
Body: pm.TextBody,
AttachNames: strings.Join(attachNames, " "),
HasAttachment: len(pm.Attachments) > 0,
Date: pm.Date,
Date: index.EffectiveDate(pm.Date, receivedAt),
Size: int64(len(raw)),
TenantID: &tid,
}