feat(PROJ-56): Last-Entzerrung für OCR und IMAP-Sync
OCR-Worker pausieren optional in konfigurierbarem Zeitfenster (paused_hours), Jobs bleiben pending statt verworfen zu werden. IMAP-Scheduler verteilt Sync-Starts via deterministischem Pro-Account-Jitter, um Lastspitzen bei vielen Postfächern mit gleichem Intervall zu vermeiden. Beides per Config opt-out, Default-Verhalten unverändert. Build + Smoke-Test auf 132 verifiziert.
This commit is contained in:
@@ -26,6 +26,11 @@ type Scheduler struct {
|
||||
mu sync.Mutex
|
||||
running map[int64]bool // in-memory guard against concurrent syncs
|
||||
|
||||
// PROJ-56: deterministic per-account jitter window in seconds. 0 disables
|
||||
// jitter. The offset for an account is derived solely from its ID so the
|
||||
// effective sync time is stable across ticks (no rand() per tick).
|
||||
jitterSeconds int
|
||||
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
@@ -39,6 +44,32 @@ func NewScheduler(store *Store, importer *Importer, logger *slog.Logger) *Schedu
|
||||
}
|
||||
}
|
||||
|
||||
// SetJitterSeconds configures the deterministic per-account sync jitter window
|
||||
// (PROJ-56). 0 disables jitter (legacy behaviour: sync exactly at interval).
|
||||
// Negative values are clamped to 0.
|
||||
func (s *Scheduler) SetJitterSeconds(seconds int) {
|
||||
if seconds < 0 {
|
||||
seconds = 0
|
||||
}
|
||||
s.jitterSeconds = seconds
|
||||
}
|
||||
|
||||
// jitterOffset returns the deterministic delay added on top of the sync
|
||||
// interval for a given account. The offset is in [0, jitterSeconds) and
|
||||
// depends only on the account ID, so it never changes between ticks.
|
||||
func (s *Scheduler) jitterOffset(accountID int64) time.Duration {
|
||||
if s.jitterSeconds <= 0 {
|
||||
return 0
|
||||
}
|
||||
// Account IDs are positive sequential integers; a simple modulo spreads
|
||||
// them evenly across the window. Use the absolute value defensively.
|
||||
id := accountID
|
||||
if id < 0 {
|
||||
id = -id
|
||||
}
|
||||
return time.Duration(id%int64(s.jitterSeconds)) * time.Second
|
||||
}
|
||||
|
||||
// SetAuditLogger wires an audit.Logger into the scheduler so that
|
||||
// UIDVALIDITY-reset events (PROJ-45) are persisted as tenant-visible
|
||||
// audit entries. Optional — when nil, only structured logs are emitted.
|
||||
@@ -123,7 +154,12 @@ func (s *Scheduler) checkAccounts(ctx context.Context) {
|
||||
lastSync = *acc.LastSyncAt
|
||||
}
|
||||
|
||||
if now.Sub(lastSync) >= interval {
|
||||
// PROJ-56: spread the actual sync start with a deterministic per-account
|
||||
// offset so accounts sharing an interval don't all poll on the same
|
||||
// minute boundary. Offset is 0 when jitter is disabled.
|
||||
dueAfter := interval + s.jitterOffset(acc.ID)
|
||||
|
||||
if now.Sub(lastSync) >= dueAfter {
|
||||
s.mu.Lock()
|
||||
s.running[acc.ID] = true
|
||||
s.mu.Unlock()
|
||||
|
||||
+58
-7
@@ -6,6 +6,7 @@ import (
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"archivmail/internal/index"
|
||||
"archivmail/internal/storage"
|
||||
@@ -32,14 +33,28 @@ type Worker struct {
|
||||
wg sync.WaitGroup
|
||||
workers int
|
||||
langs []string
|
||||
|
||||
// PROJ-56: optional local-time pause window [start, end). When the current
|
||||
// hour falls inside it, workers stop consuming the queue (jobs stay buffered
|
||||
// in the channel / as ocr_status='pending' in the DB) until it reopens.
|
||||
// nil = never pause (legacy behaviour).
|
||||
pausedHours *[2]int
|
||||
}
|
||||
|
||||
// pauseCheckInterval is how often a paused worker re-checks whether the pause
|
||||
// window has closed.
|
||||
const pauseCheckInterval = 60 * time.Second
|
||||
|
||||
// Options configures a Worker. Zero values are replaced with sensible defaults.
|
||||
type Options struct {
|
||||
QueueSize int // default 1000
|
||||
Workers int // default 2
|
||||
Langs []string // default ["deu", "eng"]
|
||||
Logger *slog.Logger
|
||||
// PausedHours optionally pauses processing during a local-time window
|
||||
// [start, end) (PROJ-56). Wrap-around windows (e.g. [22, 6]) are supported.
|
||||
// nil = never pause. The manual reprocess command leaves this nil on purpose.
|
||||
PausedHours *[2]int
|
||||
}
|
||||
|
||||
// NewWorker constructs a worker that reads mails from store, runs OCR on
|
||||
@@ -59,16 +74,37 @@ func NewWorker(store *storage.Store, idxMgr index.TenantIndexer, opts Options) *
|
||||
opts.Logger = slog.Default()
|
||||
}
|
||||
return &Worker{
|
||||
store: store,
|
||||
idxMgr: idxMgr,
|
||||
logger: opts.Logger,
|
||||
queue: make(chan Job, opts.QueueSize),
|
||||
done: make(chan struct{}),
|
||||
workers: opts.Workers,
|
||||
langs: opts.Langs,
|
||||
store: store,
|
||||
idxMgr: idxMgr,
|
||||
logger: opts.Logger,
|
||||
queue: make(chan Job, opts.QueueSize),
|
||||
done: make(chan struct{}),
|
||||
workers: opts.Workers,
|
||||
langs: opts.Langs,
|
||||
pausedHours: opts.PausedHours,
|
||||
}
|
||||
}
|
||||
|
||||
// isPaused reports whether the OCR worker should currently hold off processing
|
||||
// because the local time falls inside the configured pause window.
|
||||
// Supports wrap-around windows where start > end (e.g. [22, 6]).
|
||||
func (w *Worker) isPaused(now time.Time) bool {
|
||||
if w.pausedHours == nil {
|
||||
return false
|
||||
}
|
||||
start, end := w.pausedHours[0], w.pausedHours[1]
|
||||
if start == end {
|
||||
// Degenerate / no-op window — never pause.
|
||||
return false
|
||||
}
|
||||
h := now.Hour()
|
||||
if start < end {
|
||||
return h >= start && h < end
|
||||
}
|
||||
// Wrap-around window, e.g. [22, 6): paused 22,23,0,1,...,5.
|
||||
return h >= start || h < end
|
||||
}
|
||||
|
||||
// Submit enqueues a job. Drops with a warning if the queue is full so the
|
||||
// caller (mail intake) is never blocked.
|
||||
func (w *Worker) Submit(mailID string, tenantID *int64) {
|
||||
@@ -108,6 +144,21 @@ func (w *Worker) Stop() {
|
||||
func (w *Worker) run(ctx context.Context, id int) {
|
||||
defer w.wg.Done()
|
||||
for {
|
||||
// PROJ-56: while inside the pause window, do NOT consume the queue.
|
||||
// Jobs stay buffered in the channel (and as ocr_status='pending' in the
|
||||
// DB), so nothing is lost. We re-check periodically and still react to
|
||||
// shutdown / context cancellation immediately.
|
||||
if w.isPaused(time.Now()) {
|
||||
select {
|
||||
case <-w.done:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(pauseCheckInterval):
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case job, ok := <-w.queue:
|
||||
if !ok {
|
||||
|
||||
Reference in New Issue
Block a user