scripte/archivmail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(PROJ-30): Xapian → Manticore Search Migration

Browse Source 
- internal/index/manticore.go: ManticoreTenantManager + manticoreIndex (RT-Indizes, CGO-frei)
- internal/index/index.go: TenantIndexer Interface (Xapian + Manticore)
- internal/index/tenant_worker.go: mgr-Typ auf TenantIndexer Interface
- internal/api/server.go: idxMgr auf TenantIndexer Interface
- config/config.go: IndexConfig.ManticoreDSN Feld
- cmd/archivmail/cmd_reindex.go: reindex Subkommando
- cmd/archivmail/main.go: Manticore-Branch + reindex Case
- go.mod: github.com/go-sql-driver/mysql v1.8.1
- update.sh: Manticore auto-install, CGO_ENABLED=0, config.yml migration, auto-reindex

fix(IMAP): TCP-Deadline-Wrapper für steckengebliebene Imports
fix(auth): Email-Claim in JWT für User-Isolation
fix(search): User-Isolation via sess.Email (fail-safe)
fix(ui): Admin-Login Auth-Cache, Logout-Redirect, IMAP-Polling-Resilienz

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
sysops
2026-04-03 21:19:36 +02:00
parent e90d588e30
commit a93a843506
19 changed files with 742 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/imap/client.go
+62 -16
View File
@@ -3,17 +3,35 @@ package imap
import (
"crypto/tls"
"fmt"
"net"
"strings"
"time"
imapv2 "github.com/emersion/go-imap/v2"
"github.com/emersion/go-imap/v2/imapclient"
)
// FolderInfo describes a single IMAP folder with exclusion metadata.
type FolderInfo struct {
Name string `json:"name"`
Excluded bool `json:"excluded"`
Reason string `json:"reason,omitempty"`
const (
dialTimeout = 30 * time.Second
fetchTimeout = 5 * time.Minute // per-batch read/write deadline
)
// Conn wraps an IMAP client with the underlying net.Conn so callers
// can set per-operation deadlines to prevent indefinite blocking.
type Conn struct {
*imapclient.Client
raw net.Conn
}
// SetFetchDeadline sets a 5-minute read/write deadline on the connection.
// Call this before each fetch batch to prevent stalled imports.
func (c *Conn) SetFetchDeadline() {
_ = c.raw.SetDeadline(time.Now().Add(fetchTimeout))
}
// ClearDeadline removes any active deadline from the underlying connection.
func (c *Conn) ClearDeadline() {
_ = c.raw.SetDeadline(time.Time{})
}
// junkTrashNames lists well-known junk/trash folder names for fallback detection.
@@ -22,33 +40,61 @@ var junkTrashNames = []string{
"deleted messages", "papierkorb", "gelöschte elemente",
}
// FolderInfo describes a single IMAP folder with exclusion metadata.
type FolderInfo struct {
Name string `json:"name"`
Excluded bool `json:"excluded"`
Reason string `json:"reason,omitempty"`
}
// Connect establishes an IMAP client connection using the specified TLS mode.
func Connect(host string, port int, tlsMode string) (*imapclient.Client, error) {
// Returns a Conn that exposes the underlying net.Conn for deadline management.
func Connect(host string, port int, tlsMode string) (*Conn, error) {
addr := fmt.Sprintf("%s:%d", host, port)
switch tlsMode {
case "ssl":
c, err := imapclient.DialTLS(addr, &imapclient.Options{
TLSConfig: &tls.Config{ServerName: host},
})
dialer := &tls.Dialer{
NetDialer: &net.Dialer{Timeout: dialTimeout},
Config: &tls.Config{ServerName: host},
}
raw, err := dialer.Dial("tcp", addr)
if err != nil {
return nil, fmt.Errorf("imap connect ssl: %w", err)
}
return c, nil
c, err := imapclient.New(raw, nil)
if err != nil {
raw.Close()
return nil, fmt.Errorf("imap client ssl: %w", err)
}
return &Conn{Client: c, raw: raw}, nil
case "starttls":
c, err := imapclient.DialStartTLS(addr, &imapclient.Options{
TLSConfig: &tls.Config{ServerName: host},
})
raw, err := net.DialTimeout("tcp", addr, dialTimeout)
if err != nil {
return nil, fmt.Errorf("imap connect starttls: %w", err)
}
return c, nil
c, err := imapclient.New(raw, &imapclient.Options{
TLSConfig: &tls.Config{ServerName: host},
})
if err != nil {
raw.Close()
return nil, fmt.Errorf("imap client starttls: %w", err)
}
return &Conn{Client: c, raw: raw}, nil
case "none":
c, err := imapclient.DialInsecure(addr, nil)
raw, err := net.DialTimeout("tcp", addr, dialTimeout)
if err != nil {
return nil, fmt.Errorf("imap connect plain: %w", err)
}
return c, nil
c, err := imapclient.New(raw, nil)
if err != nil {
raw.Close()
return nil, fmt.Errorf("imap client plain: %w", err)
}
return &Conn{Client: c, raw: raw}, nil
default:
return nil, fmt.Errorf("imap: unknown tls mode %q", tlsMode)
}
internal/imap/importer.go
+11 -10
View File
@@ -23,7 +23,6 @@ type Importer struct {
mailStore *storage.Store
idx index.Indexer
logger *slog.Logger
TenantID *int64 // optional tenant assignment for stored mails
}
// NewImporter creates a new Importer wired to the storage and index backends.
@@ -88,7 +87,7 @@ func (imp *Importer) doImport(ctx context.Context, acc *Account, password string
}
// List all folders
folders, err := ListFolders(c)
folders, err := ListFolders(c.Client)
if err != nil {
return 0, fmt.Errorf("list folders: %w", err)
}
@@ -159,11 +158,13 @@ func (imp *Importer) doImport(ctx context.Context, acc *Account, password string
}
batch := uids[i:end]
count, err := imp.fetchBatch(ctx, c, batch, log)
// Set per-batch deadline to prevent indefinite blocking on stalled connections.
c.SetFetchDeadline()
count, err := imp.fetchBatch(ctx, c.Client, batch, acc.TenantID, log)
c.ClearDeadline()
if err != nil {
log.Error("batch fetch error", "folder", folder, "offset", i, "err", err)
// Continue with the next batch rather than aborting entirely
continue
log.Error("batch fetch error — aborting import", "folder", folder, "offset", i, "err", err)
return imported, fmt.Errorf("fetch batch %d in %q: %w", i, folder, err)
}
imported += count
@@ -177,7 +178,7 @@ func (imp *Importer) doImport(ctx context.Context, acc *Account, password string
}
// fetchBatch fetches and stores a batch of messages by UID.
func (imp *Importer) fetchBatch(ctx context.Context, c *imapclient.Client, uids []imapv2.UID, log *slog.Logger) (int, error) {
func (imp *Importer) fetchBatch(ctx context.Context, c *imapclient.Client, uids []imapv2.UID, tenantID *int64, log *slog.Logger) (int, error) {
if len(uids) == 0 {
return 0, nil
}
@@ -212,7 +213,7 @@ func (imp *Importer) fetchBatch(ctx context.Context, c *imapclient.Client, uids
continue
}
if err := imp.storeAndIndex(raw, log); err != nil {
if err := imp.storeAndIndex(raw, tenantID, log); err != nil {
log.Warn("failed to store/index message", "err", err)
continue
}
@@ -229,10 +230,10 @@ func (imp *Importer) fetchBatch(ctx context.Context, c *imapclient.Client, uids
}
// storeAndIndex saves a raw email to storage and indexes it.
func (imp *Importer) storeAndIndex(raw []byte, log *slog.Logger) error {
func (imp *Importer) storeAndIndex(raw []byte, tenantID *int64, log *slog.Logger) error {
ctx := context.Background()
// Save to file storage (deduplicates by SHA256 automatically)
id, err := imp.mailStore.Save(ctx, raw, time.Now(), imp.TenantID)
id, err := imp.mailStore.Save(ctx, raw, time.Now(), tenantID)
if err != nil {
return fmt.Errorf("save: %w", err)
}
internal/imap/scheduler.go
+7 -4
View File
@@ -207,7 +207,7 @@ func (s *Scheduler) doSync(ctx context.Context, accountID int64) (int, uint32, e
return 0, 0, fmt.Errorf("imap scheduler: login: %w", err)
}
folders, err := ListFolders(c)
folders, err := ListFolders(c.Client)
if err != nil {
return 0, 0, fmt.Errorf("imap scheduler: list folders: %w", err)
}
@@ -247,7 +247,7 @@ func (s *Scheduler) doSync(ctx context.Context, accountID int64) (int, uint32, e
// syncFolder syncs new messages from a single IMAP folder.
func (s *Scheduler) syncFolder(
ctx context.Context,
c *imapclient.Client,
c *Conn,
acc *Account,
folder string,
log *slog.Logger,
@@ -298,7 +298,9 @@ func (s *Scheduler) syncFolder(
}
batch := uids[i:end]
count, batchMaxUID, err := s.fetchSyncBatch(c, batch, log)
c.SetFetchDeadline()
count, batchMaxUID, err := s.fetchSyncBatch(c.Client, batch, acc.TenantID, log)
c.ClearDeadline()
if err != nil {
log.Warn("imap scheduler: batch error, continuing",
"folder", folder, "offset", i, "err", err)
@@ -320,6 +322,7 @@ func (s *Scheduler) syncFolder(
func (s *Scheduler) fetchSyncBatch(
c *imapclient.Client,
uids []imapv2.UID,
tenantID *int64,
log *slog.Logger,
) (int, uint32, error) {
if len(uids) == 0 {
@@ -367,7 +370,7 @@ func (s *Scheduler) fetchSyncBatch(
}
if len(raw) > 0 {
if err := s.importer.storeAndIndex(raw, log); err != nil {
if err := s.importer.storeAndIndex(raw, tenantID, log); err != nil {
log.Warn("imap scheduler: store/index failed", "err", err)
} else {
imported++
internal/imap/store.go
+7 -3
View File
@@ -40,6 +40,9 @@ type Account struct {
SyncRunning bool `json:"sync_running"`
SyncStatus string `json:"sync_status"`
SyncErrorMsg string `json:"sync_error_msg"`
// Tenant assignment — mails imported from this account are tagged with this tenant.
TenantID *int64 `json:"tenant_id,omitempty"`
}
// Store manages IMAP account persistence in PostgreSQL.
@@ -71,7 +74,7 @@ CREATE TABLE IF NOT EXISTS imap_accounts (
CREATE INDEX IF NOT EXISTS idx_imap_accounts_owner ON imap_accounts (owner);
`
// migrationSQL adds the PROJ-8 sync columns if they do not yet exist.
// migrationSQL adds columns that may not exist in older installations.
const migrationSQL = `
ALTER TABLE imap_accounts ADD COLUMN IF NOT EXISTS sync_interval_min INTEGER NOT NULL DEFAULT 0;
ALTER TABLE imap_accounts ADD COLUMN IF NOT EXISTS last_sync_at TIMESTAMPTZ;
@@ -80,6 +83,7 @@ ALTER TABLE imap_accounts ADD COLUMN IF NOT EXISTS last_uid BIGINT NOT NULL DEFA
ALTER TABLE imap_accounts ADD COLUMN IF NOT EXISTS sync_running BOOLEAN NOT NULL DEFAULT FALSE;
ALTER TABLE imap_accounts ADD COLUMN IF NOT EXISTS sync_status TEXT NOT NULL DEFAULT '';
ALTER TABLE imap_accounts ADD COLUMN IF NOT EXISTS sync_error_msg TEXT NOT NULL DEFAULT '';
ALTER TABLE imap_accounts ADD COLUMN IF NOT EXISTS tenant_id INTEGER REFERENCES tenants(id);
`
// New creates a new Store, connects to PostgreSQL, and runs the migration.
@@ -138,7 +142,7 @@ const selectColumns = ` id, owner, name, host, port, tls, username, excluded_fol
status, error_msg, last_import_at, last_import_count,
progress_current, progress_total, created_at,
sync_interval_min, last_sync_at, last_sync_count, last_uid,
sync_running, sync_status, sync_error_msg `
sync_running, sync_status, sync_error_msg, tenant_id `
// scanner abstracts pgx.Row and pgx.Rows — both expose Scan(...any) error.
type scanner interface {
@@ -152,7 +156,7 @@ func scanRow(row scanner) (Account, error) {
&a.ExcludedFolders, &a.Status, &a.ErrorMsg, &a.LastImportAt,
&a.LastImportCount, &a.ProgressCurrent, &a.ProgressTotal, &a.CreatedAt,
&a.SyncIntervalMin, &a.LastSyncAt, &a.LastSyncCount, &a.LastUID,
&a.SyncRunning, &a.SyncStatus, &a.SyncErrorMsg,
&a.SyncRunning, &a.SyncStatus, &a.SyncErrorMsg, &a.TenantID,
)
return a, err
}