feat(PROJ-5): AES-256-GCM Verschlüsselung, PostgreSQL Metadaten, Async Index Worker
- Storage: AES-256-GCM Verschlüsselung (keyfile, graceful fallback bei fehlendem Key) - Storage: PostgreSQL emails-Tabelle mit Auto-Migration - Storage: Save/Delete/Stats/FirstAndLastMail nutzen DB wenn verfügbar - Index: Async IndexWorker (Go-Channel, Queue 1000, non-blocking Submit) - SMTP: IndexCallback für async Indexierung nach Mail-Eingang - main: Backfill beim Start (40 Mails migriert + indexiert) - Bestehende Mails werden transparent entschlüsselt (Fallback auf Raw) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
sysops
@@ -64,11 +64,17 @@ func runExport(args []string) {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
mailStore, err := storage.New(cfg.Storage.StorePath)
|
||||
storeCfg := storage.Config{
|
||||
Dir: cfg.Storage.StorePath,
|
||||
Keyfile: cfg.Storage.Keyfile,
|
||||
DSN: cfg.Database.DSN(),
|
||||
}
|
||||
mailStore, err := storage.New(storeCfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: storage init: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer mailStore.Close()
|
||||
|
||||
batchSize := cfg.Index.BatchSize
|
||||
if batchSize <= 0 {
|
||||
|
||||
@@ -56,11 +56,17 @@ func runImport(args []string) {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
mailStore, err := storage.New(cfg.Storage.StorePath)
|
||||
storeCfg := storage.Config{
|
||||
Dir: cfg.Storage.StorePath,
|
||||
Keyfile: cfg.Storage.Keyfile,
|
||||
DSN: cfg.Database.DSN(),
|
||||
}
|
||||
mailStore, err := storage.New(storeCfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: storage init: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer mailStore.Close()
|
||||
|
||||
batchSize := cfg.Index.BatchSize
|
||||
if batchSize <= 0 {
|
||||
|
||||
+119
-6
@@ -8,6 +8,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -20,6 +21,7 @@ import (
|
||||
"github.com/archivmail/internal/smtpd"
|
||||
"github.com/archivmail/internal/storage"
|
||||
"github.com/archivmail/internal/userstore"
|
||||
"github.com/archivmail/pkg/mailparser"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -54,12 +56,18 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Storage
|
||||
mailStore, err := storage.New(cfg.Storage.StorePath)
|
||||
// Storage with encryption + DB metadata
|
||||
storeCfg := storage.Config{
|
||||
Dir: cfg.Storage.StorePath,
|
||||
Keyfile: cfg.Storage.Keyfile,
|
||||
DSN: cfg.Database.DSN(),
|
||||
}
|
||||
mailStore, err := storage.New(storeCfg)
|
||||
if err != nil {
|
||||
logger.Error("storage init failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer mailStore.Close()
|
||||
|
||||
// Index
|
||||
indexBackend := cfg.Index.Backend
|
||||
@@ -77,6 +85,15 @@ func main() {
|
||||
}
|
||||
defer idx.Close()
|
||||
|
||||
// Async index worker
|
||||
asyncQueueSize := cfg.Index.AsyncQueueSize
|
||||
if asyncQueueSize <= 0 {
|
||||
asyncQueueSize = 1000
|
||||
}
|
||||
worker := index.NewWorker(idx, asyncQueueSize, logger)
|
||||
worker.Start()
|
||||
defer worker.Stop()
|
||||
|
||||
// User store
|
||||
users, err := userstore.New(cfg.Database.DSN())
|
||||
if err != nil {
|
||||
@@ -118,11 +135,14 @@ func main() {
|
||||
Handler: srv,
|
||||
}
|
||||
|
||||
// Start SMTP daemon
|
||||
// Start SMTP daemon with index worker integration
|
||||
if cfg.SMTP.Bind == "" {
|
||||
cfg.SMTP.Bind = fmt.Sprintf(":%d", cfg.Server.SMTPPort)
|
||||
}
|
||||
smtpDaemon := smtpd.New(cfg.SMTP, mailStore, logger)
|
||||
smtpDaemon.SetIndexCallback(func(raw []byte, id string) {
|
||||
submitToWorker(worker, mailStore, raw, id, logger)
|
||||
})
|
||||
if err := smtpDaemon.Start(); err != nil {
|
||||
logger.Error("SMTP daemon failed to start", "err", err)
|
||||
os.Exit(1)
|
||||
@@ -132,7 +152,7 @@ func main() {
|
||||
// Wire SMTP daemon into API server for status endpoint
|
||||
srv.SetSMTPDaemon(smtpDaemon)
|
||||
|
||||
// IMAP store + importer
|
||||
// IMAP store + importer (wired to use async worker)
|
||||
imapSt, err := imapstore.New(cfg.Database.DSN(), cfg.API.Secret)
|
||||
if err != nil {
|
||||
logger.Error("imap store init failed", "err", err)
|
||||
@@ -142,6 +162,9 @@ func main() {
|
||||
imapImp := imapstore.NewImporter(imapSt, mailStore, idx, logger)
|
||||
srv.SetImap(imapSt, imapImp)
|
||||
|
||||
// Backfill in background: migrate existing files into DB metadata + re-index
|
||||
go runBackfill(context.Background(), mailStore, idx, worker, logger)
|
||||
|
||||
// Start HTTP API
|
||||
go func() {
|
||||
logger.Info("starting API server", "addr", bind)
|
||||
@@ -161,6 +184,98 @@ func main() {
|
||||
httpServer.Shutdown(ctx)
|
||||
}
|
||||
|
||||
// submitToWorker parses a raw email and submits it to the async index worker.
|
||||
func submitToWorker(worker *index.IndexWorker, store *storage.Store, raw []byte, id string, logger *slog.Logger) {
|
||||
pm, err := mailparser.Parse(raw)
|
||||
if err != nil {
|
||||
logger.Warn("index: parse failed, skipping indexing", "id", id, "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
var attachNames []string
|
||||
for _, a := range pm.Attachments {
|
||||
if a.Filename != "" {
|
||||
attachNames = append(attachNames, a.Filename)
|
||||
}
|
||||
}
|
||||
|
||||
doc := index.MailDocument{
|
||||
ID: id,
|
||||
From: pm.From,
|
||||
To: strings.Join(pm.To, ", "),
|
||||
Subject: pm.Subject,
|
||||
Body: pm.TextBody,
|
||||
AttachNames: strings.Join(attachNames, " "),
|
||||
HasAttachment: len(pm.Attachments) > 0,
|
||||
Date: pm.Date,
|
||||
Size: int64(len(raw)),
|
||||
}
|
||||
|
||||
worker.Submit(doc)
|
||||
|
||||
// Mark as indexed in DB
|
||||
if err := store.SetIndexedAt(context.Background(), id); err != nil {
|
||||
logger.Warn("index: set indexed_at failed", "id", id, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// runBackfill walks the store, inserts missing DB metadata, and indexes
|
||||
// emails that have not yet been indexed.
|
||||
func runBackfill(ctx context.Context, store *storage.Store, idx index.Indexer, worker *index.IndexWorker, logger *slog.Logger) {
|
||||
logger.Info("backfill: starting")
|
||||
|
||||
count := 0
|
||||
needIndex := 0
|
||||
errCount := 0
|
||||
|
||||
err := store.WalkStore(ctx, func(id string) error {
|
||||
count++
|
||||
|
||||
raw, err := store.Load(id)
|
||||
if err != nil {
|
||||
logger.Warn("backfill: load failed", "id", id, "err", err)
|
||||
errCount++
|
||||
return nil
|
||||
}
|
||||
|
||||
pm, err := mailparser.Parse(raw)
|
||||
if err != nil {
|
||||
logger.Warn("backfill: parse failed", "id", id, "err", err)
|
||||
errCount++
|
||||
return nil
|
||||
}
|
||||
|
||||
// Upsert metadata into DB
|
||||
if err := store.SaveMeta(ctx, id, pm, len(raw)); err != nil {
|
||||
logger.Warn("backfill: save meta failed", "id", id, "err", err)
|
||||
}
|
||||
|
||||
// Check if already indexed
|
||||
alreadyIndexed, err := store.IsIndexed(ctx, id)
|
||||
if err != nil {
|
||||
logger.Warn("backfill: check indexed failed", "id", id, "err", err)
|
||||
}
|
||||
|
||||
if !alreadyIndexed {
|
||||
needIndex++
|
||||
submitToWorker(worker, store, raw, id, logger)
|
||||
}
|
||||
|
||||
if count%100 == 0 {
|
||||
logger.Info("backfill: progress", "processed", count, "need_index", needIndex, "errors", errCount)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
logger.Error("backfill failed", "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Info("backfill: complete", "total", count, "submitted_for_index", needIndex, "errors", errCount)
|
||||
}
|
||||
|
||||
// seedDefaultUsers creates default admin and auditor accounts if no users exist yet.
|
||||
func seedDefaultUsers(users *userstore.Store, logger *slog.Logger) error {
|
||||
all, err := users.List("")
|
||||
@@ -183,5 +298,3 @@ func seedDefaultUsers(users *userstore.Store, logger *slog.Logger) error {
|
||||
logger.Warn("default users created — change passwords immediately!", "admin", "admin", "auditor", "auditor")
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user