scripte/archivmail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(PROJ-45): IMAP Per-Folder UID-Tracking + UIDVALIDITY-Check

Browse Source 
- FolderState: GetFolderState, UpsertFolderState, ListFolderStates, DecideResync
- syncFolder nutzt per-folder UID-Tracking statt globalem highest_uid
- UIDVALIDITY-Check loest automatisch Full-Resync aus
- imap_folder_state Tabelle in initSchema (CREATE TABLE IF NOT EXISTS)
- SetAuditLogger in main.go verdrahtet

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
sysops
2026-05-11 10:49:14 +02:00
parent 16013e8b66
commit 4151b6f8c5
5 changed files with 338 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/imap/scheduler.go
+114 -16
View File
@@ -9,6 +9,8 @@ import (
"sync"
"time"
"archivmail/internal/audit"
imapv2 "github.com/emersion/go-imap/v2"
"github.com/emersion/go-imap/v2/imapclient"
)
@@ -19,6 +21,7 @@ type Scheduler struct {
store *Store
importer *Importer
logger *slog.Logger
audlog *audit.Logger // PROJ-45: UIDVALIDITY-reset events are tenant-visible audit entries
mu sync.Mutex
running map[int64]bool // in-memory guard against concurrent syncs
@@ -36,6 +39,13 @@ func NewScheduler(store *Store, importer *Importer, logger *slog.Logger) *Schedu
}
}
// SetAuditLogger wires an audit.Logger into the scheduler so that
// UIDVALIDITY-reset events (PROJ-45) are persisted as tenant-visible
// audit entries. Optional — when nil, only structured logs are emitted.
func (s *Scheduler) SetAuditLogger(a *audit.Logger) {
s.audlog = a
}
// Start launches the background scheduler goroutine.
// Call Stop to shut it down gracefully.
func (s *Scheduler) Start() {
@@ -150,6 +160,8 @@ func (s *Scheduler) runSyncWithRetry(ctx context.Context, accountID int64) {
lastErr error
)
startedAt := time.Now()
for attempt := 0; attempt < 3; attempt++ {
if attempt > 0 {
backoff := syncBackoff[attempt-1]
@@ -172,15 +184,18 @@ func (s *Scheduler) runSyncWithRetry(ctx context.Context, accountID int64) {
"account_id", accountID, "attempt", attempt+1, "err", lastErr)
}
durationMs := time.Since(startedAt).Milliseconds()
if lastErr != nil {
s.logger.Error("imap scheduler: sync failed after all retries",
"account_id", accountID, "err", lastErr)
"account_id", accountID, "err", lastErr, "duration_ms", durationMs)
_ = s.store.UpdateSyncResult(ctx, accountID, "error", lastErr.Error(), 0, lastUID)
return
}
s.logger.Info("imap scheduler: sync completed",
"account_id", accountID, "imported", count, "last_uid", lastUID)
"account_id", accountID, "imported", count, "last_uid", lastUID,
"duration_ms", durationMs)
_ = s.store.UpdateSyncResult(ctx, accountID, "ok", "", count, lastUID)
}
@@ -220,8 +235,9 @@ func (s *Scheduler) doSync(ctx context.Context, accountID int64) (int, uint32, e
log := s.logger.With("component", "imap-scheduler", "account_id", accountID)
var (
totalImported int
maxUID uint32 = acc.LastUID
totalImported int
maxUID uint32
foldersSynced int
)
for _, folder := range folders {
@@ -235,16 +251,30 @@ func (s *Scheduler) doSync(ctx context.Context, accountID int64) (int, uint32, e
continue
}
foldersSynced++
totalImported += count
if folderMaxUID > maxUID {
maxUID = folderMaxUID
}
}
// Account-level summary (PROJ-45 acceptance criterion).
// duration_ms is intentionally not added here — it's logged by runSyncWithRetry
// which owns the retry-wrapped wall clock.
log.Info("account sync complete",
"folders_synced", foldersSynced,
"new_messages_total", totalImported)
return totalImported, maxUID, nil
}
// syncFolder syncs new messages from a single IMAP folder.
//
// PROJ-45: Uses per-folder UID tracking from imap_folder_state instead of the
// legacy account-global imap_accounts.last_uid. UIDVALIDITY is read from the
// SELECT response and compared against the stored value — on mismatch the
// folder is fully resynced in this same iteration (PROJ-32 message-id dedup
// blocks duplicates).
func (s *Scheduler) syncFolder(
ctx context.Context,
c *Conn,
@@ -252,15 +282,59 @@ func (s *Scheduler) syncFolder(
folder string,
log *slog.Logger,
) (int, uint32, error) {
if _, err := c.Select(folder, nil).Wait(); err != nil {
// Load per-folder state BEFORE selecting — we need the stored uid_validity
// to compare against the value the server reports in the SELECT response.
state, err := s.store.GetFolderState(ctx, acc.ID, folder)
if err != nil {
return 0, 0, fmt.Errorf("imap scheduler: get folder state %q: %w", folder, err)
}
selectData, err := c.Select(folder, nil).Wait()
if err != nil {
return 0, 0, fmt.Errorf("imap scheduler: select %q: %w", folder, err)
}
// Snapshot the server-reported UIDVALIDITY. A value of 0 is treated as
// "unknown / server bug" — we keep the existing high-water mark instead
// of forcing a needless full resync.
serverUIDValidity := uint32(0)
if selectData != nil {
serverUIDValidity = selectData.UIDValidity
}
effectiveLastUID := state.LastUID
switch {
case serverUIDValidity == 0:
log.Warn("uidvalidity zero from server, keeping stored last_uid",
"folder", folder, "stored_uid_validity", state.UIDValidity)
case state.UIDValidity != 0 && state.UIDValidity != serverUIDValidity:
// Mailbox was rebuilt server-side — UIDs restart at 1. We must
// resync the entire folder from scratch. PROJ-32 message-id
// deduplication prevents duplicate writes.
log.Warn("uidvalidity change",
"account_id", acc.ID,
"folder", folder,
"old", state.UIDValidity,
"new", serverUIDValidity)
if s.audlog != nil {
s.audlog.Log(audit.Entry{
EventType: "imap_uidvalidity_reset",
Username: acc.Owner,
Success: true,
Detail: fmt.Sprintf("account=%d folder=%q old_uidvalidity=%d new_uidvalidity=%d",
acc.ID, folder, state.UIDValidity, serverUIDValidity),
})
}
effectiveLastUID = 0
}
var uids []imapv2.UID
if acc.LastUID > 0 {
// Incremental: only messages with UID > lastUID.
minUID := imapv2.UID(acc.LastUID + 1)
if effectiveLastUID > 0 {
// Incremental: only messages with UID > effectiveLastUID.
minUID := imapv2.UID(effectiveLastUID + 1)
criteria := &imapv2.SearchCriteria{
UID: []imapv2.UIDSet{
{imapv2.UIDRange{Start: minUID, Stop: 0}},
@@ -272,7 +346,7 @@ func (s *Scheduler) syncFolder(
}
uids = searchData.AllUIDs()
} else {
// First sync: fetch everything.
// First sync or post-UIDVALIDITY-reset: fetch everything.
searchData, err := c.UIDSearch(&imapv2.SearchCriteria{}, nil).Wait()
if err != nil {
return 0, 0, fmt.Errorf("imap scheduler: uid search full %q: %w", folder, err)
@@ -280,17 +354,35 @@ func (s *Scheduler) syncFolder(
uids = searchData.AllUIDs()
}
if len(uids) == 0 {
return 0, 0, nil
}
log.Info("syncing folder", "folder", folder, "new_messages", len(uids))
var (
imported int
maxUID uint32
maxUID uint32 = effectiveLastUID
)
// PROJ-45: even when there are 0 new UIDs, persist the (possibly new)
// uid_validity so a later UIDVALIDITY change can still be detected.
defer func() {
newState := FolderState{
AccountID: acc.ID,
Folder: folder,
LastUID: maxUID,
UIDValidity: serverUIDValidity,
}
if err := s.store.UpsertFolderState(ctx, newState); err != nil {
log.Warn("imap scheduler: persist folder state failed",
"folder", folder, "err", err)
}
}()
if len(uids) == 0 {
log.Info("folder synced",
"folder", folder,
"new_messages", 0,
"total_messages", 0,
"uid_validity", serverUIDValidity)
return 0, maxUID, nil
}
for i := 0; i < len(uids); i += batchSize {
end := i + batchSize
if end > len(uids) {
@@ -313,6 +405,12 @@ func (s *Scheduler) syncFolder(
}
}
log.Info("folder synced",
"folder", folder,
"new_messages", imported,
"total_messages", len(uids),
"uid_validity", serverUIDValidity)
return imported, maxUID, nil
}