feat(PROJ-45): IMAP per-folder UID-tracking, UIDVALIDITY-check + reindex OCR protection
- scheduler.go: BUG-1 fix — preserve stored uid_validity when server returns 0 - scheduler.go: BUG-2 fix — replace inline switch with DecideResync() call - scheduler.go: SetAuditLogger wired; imap_uidvalidity_reset audit event - cmd_reindex.go: read existing attachment_text before IndexSync to prevent Manticore REPLACE INTO from wiping OCR text written by the OCR worker Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -129,6 +129,19 @@ func runReindex(args []string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
idx := idxMgr.ForTenant(tenantID)
|
idx := idxMgr.ForTenant(tenantID)
|
||||||
|
|
||||||
|
// Preserve existing OCR text — IndexSync uses REPLACE which would
|
||||||
|
// otherwise wipe attachment_text written by the OCR worker.
|
||||||
|
// Only read from the index when the DB confirms OCR has run (ocr_chars>0).
|
||||||
|
_, ocrChars, _ := mailStore.GetOCRMeta(ctx, id)
|
||||||
|
if ocrChars > 0 {
|
||||||
|
if reader, ok := idx.(index.AttachmentTextReader); ok {
|
||||||
|
if existing, err := reader.GetAttachmentText(id); err == nil && existing != "" {
|
||||||
|
doc.AttachmentText = existing
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if err := idx.IndexSync(doc); err != nil {
|
if err := idx.IndexSync(doc); err != nil {
|
||||||
logger.Warn("reindex: index failed", "id", id, "err", err)
|
logger.Warn("reindex: index failed", "id", id, "err", err)
|
||||||
errors++
|
errors++
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ Ein selbst gehostetes, unternehmenstaugliches Mail-Archiv-System, das E-Mails au
|
|||||||
| P2 | Audit-Log & Compliance-Berichte | Planned |
|
| P2 | Audit-Log & Compliance-Berichte | Planned |
|
||||||
| P2 | E-Mail-Export (EML/PDF) | Planned |
|
| P2 | E-Mail-Export (EML/PDF) | Planned |
|
||||||
| P2 | REST API für externe CRM-Anbindung | Planned |
|
| P2 | REST API für externe CRM-Anbindung | Planned |
|
||||||
|
| P1 | IMAP Per-Folder UID-Tracking + UIDVALIDITY-Check (Reliability-Fix) | Planned |
|
||||||
|
|
||||||
## Success Metrics
|
## Success Metrics
|
||||||
- Import und Indexierung von 100.000+ E-Mails ohne Performanceprobleme
|
- Import und Indexierung von 100.000+ E-Mails ohne Performanceprobleme
|
||||||
|
|||||||
+3
-1
@@ -60,7 +60,9 @@
|
|||||||
| PROJ-41 | Dashboard Zeitreihe + Speicherprognose | Deployed | [PROJ-41](PROJ-41-dashboard-zeitreihe.md) | 2026-04-05 |
|
| PROJ-41 | Dashboard Zeitreihe + Speicherprognose | Deployed | [PROJ-41](PROJ-41-dashboard-zeitreihe.md) | 2026-04-05 |
|
||||||
| PROJ-42 | Gespeicherte Suchanfragen | Deployed | [PROJ-42](PROJ-42-gespeicherte-suchanfragen.md) | 2026-04-05 |
|
| PROJ-42 | Gespeicherte Suchanfragen | Deployed | [PROJ-42](PROJ-42-gespeicherte-suchanfragen.md) | 2026-04-05 |
|
||||||
| PROJ-43 | Automatische Archivierungsregeln | Planned | [PROJ-43](PROJ-43-archivierungsregeln.md) | 2026-04-05 |
|
| PROJ-43 | Automatische Archivierungsregeln | Planned | [PROJ-43](PROJ-43-archivierungsregeln.md) | 2026-04-05 |
|
||||||
|
| PROJ-44 | OCR-GUI-Integration (Status, Download, Such-Highlight) | In Review | [PROJ-44](PROJ-44-ocr-gui-integration.md) | 2026-05-08 |
|
||||||
|
| PROJ-45 | IMAP Per-Folder UID-Tracking + UIDVALIDITY-Check | Deployed | [PROJ-45](PROJ-45-imap-folder-uid-tracking.md) | 2026-05-11 |
|
||||||
|
|
||||||
<!-- Add features above this line -->
|
<!-- Add features above this line -->
|
||||||
|
|
||||||
## Next Available ID: PROJ-44
|
## Next Available ID: PROJ-46
|
||||||
|
|||||||
@@ -302,13 +302,16 @@ func (s *Scheduler) syncFolder(
|
|||||||
serverUIDValidity = selectData.UIDValidity
|
serverUIDValidity = selectData.UIDValidity
|
||||||
}
|
}
|
||||||
|
|
||||||
effectiveLastUID := state.LastUID
|
// BUG-2 fix: use DecideResync instead of inline switch to keep the
|
||||||
|
// branching logic in one place and avoid drift with the unit-tested path.
|
||||||
|
decision := DecideResync(state.LastUID, state.UIDValidity, serverUIDValidity)
|
||||||
|
effectiveLastUID := decision.EffectiveStart
|
||||||
|
|
||||||
switch {
|
if serverUIDValidity == 0 {
|
||||||
case serverUIDValidity == 0:
|
|
||||||
log.Warn("uidvalidity zero from server, keeping stored last_uid",
|
log.Warn("uidvalidity zero from server, keeping stored last_uid",
|
||||||
"folder", folder, "stored_uid_validity", state.UIDValidity)
|
"folder", folder, "stored_uid_validity", state.UIDValidity)
|
||||||
case state.UIDValidity != 0 && state.UIDValidity != serverUIDValidity:
|
}
|
||||||
|
if decision.UIDValidReset {
|
||||||
// Mailbox was rebuilt server-side — UIDs restart at 1. We must
|
// Mailbox was rebuilt server-side — UIDs restart at 1. We must
|
||||||
// resync the entire folder from scratch. PROJ-32 message-id
|
// resync the entire folder from scratch. PROJ-32 message-id
|
||||||
// deduplication prevents duplicate writes.
|
// deduplication prevents duplicate writes.
|
||||||
@@ -327,7 +330,6 @@ func (s *Scheduler) syncFolder(
|
|||||||
acc.ID, folder, state.UIDValidity, serverUIDValidity),
|
acc.ID, folder, state.UIDValidity, serverUIDValidity),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
effectiveLastUID = 0
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var uids []imapv2.UID
|
var uids []imapv2.UID
|
||||||
@@ -359,14 +361,22 @@ func (s *Scheduler) syncFolder(
|
|||||||
maxUID uint32 = effectiveLastUID
|
maxUID uint32 = effectiveLastUID
|
||||||
)
|
)
|
||||||
|
|
||||||
// PROJ-45: even when there are 0 new UIDs, persist the (possibly new)
|
// BUG-1 fix: when the server returns UIDVALIDITY=0 (misbehaving server),
|
||||||
// uid_validity so a later UIDVALIDITY change can still be detected.
|
// keep the stored valid value so future mismatch detection still works.
|
||||||
|
// Writing 0 would silently disable the UIDVALIDITY guard for this folder.
|
||||||
|
persistedValidity := serverUIDValidity
|
||||||
|
if serverUIDValidity == 0 {
|
||||||
|
persistedValidity = state.UIDValidity
|
||||||
|
}
|
||||||
|
|
||||||
|
// PROJ-45: even when there are 0 new UIDs, persist the uid_validity
|
||||||
|
// so a later UIDVALIDITY change can still be detected.
|
||||||
defer func() {
|
defer func() {
|
||||||
newState := FolderState{
|
newState := FolderState{
|
||||||
AccountID: acc.ID,
|
AccountID: acc.ID,
|
||||||
Folder: folder,
|
Folder: folder,
|
||||||
LastUID: maxUID,
|
LastUID: maxUID,
|
||||||
UIDValidity: serverUIDValidity,
|
UIDValidity: persistedValidity,
|
||||||
}
|
}
|
||||||
if err := s.store.UpsertFolderState(ctx, newState); err != nil {
|
if err := s.store.UpsertFolderState(ctx, newState); err != nil {
|
||||||
log.Warn("imap scheduler: persist folder state failed",
|
log.Warn("imap scheduler: persist folder state failed",
|
||||||
|
|||||||
Reference in New Issue
Block a user