feat(PROJ-17): Admin Dashboard Systemauslastung immer anzeigen

- Systemauslastungs-Sektion wird immer gerendert (nicht nur bei Erfolg) - Fehlermeldung wenn /api/admin/system/stats nicht erreichbar ist - Feature-Status auf In Review gesetzt Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-14 11:43:19 +01:00
parent a893084a88
commit d360c9a5ba
68 changed files with 11938 additions and 435 deletions
@@ -0,0 +1,99 @@
+package imap
+
+import (
+	"crypto/tls"
+	"fmt"
+	"strings"
+
+	imapv2 "github.com/emersion/go-imap/v2"
+	"github.com/emersion/go-imap/v2/imapclient"
+)
+
+// FolderInfo describes a single IMAP folder with exclusion metadata.
+type FolderInfo struct {
+	Name     string `json:"name"`
+	Excluded bool   `json:"excluded"`
+	Reason   string `json:"reason,omitempty"`
+}
+
+// junkTrashNames lists well-known junk/trash folder names for fallback detection.
+var junkTrashNames = []string{
+	"junk", "spam", "trash", "deleted items",
+	"deleted messages", "papierkorb", "gelöschte elemente",
+}
+
+// Connect establishes an IMAP client connection using the specified TLS mode.
+func Connect(host string, port int, tlsMode string) (*imapclient.Client, error) {
+	addr := fmt.Sprintf("%s:%d", host, port)
+
+	switch tlsMode {
+	case "ssl":
+		c, err := imapclient.DialTLS(addr, &imapclient.Options{
+			TLSConfig: &tls.Config{ServerName: host},
+		})
+		if err != nil {
+			return nil, fmt.Errorf("imap connect ssl: %w", err)
+		}
+		return c, nil
+	case "starttls":
+		c, err := imapclient.DialStartTLS(addr, &imapclient.Options{
+			TLSConfig: &tls.Config{ServerName: host},
+		})
+		if err != nil {
+			return nil, fmt.Errorf("imap connect starttls: %w", err)
+		}
+		return c, nil
+	case "none":
+		c, err := imapclient.DialInsecure(addr, nil)
+		if err != nil {
+			return nil, fmt.Errorf("imap connect plain: %w", err)
+		}
+		return c, nil
+	default:
+		return nil, fmt.Errorf("imap: unknown tls mode %q", tlsMode)
+	}
+}
+
+// ListFolders retrieves all mailbox folders and detects junk/trash folders.
+func ListFolders(c *imapclient.Client) ([]FolderInfo, error) {
+	listCmd := c.List("", "*", nil)
+	mailboxes, err := listCmd.Collect()
+	if err != nil {
+		return nil, fmt.Errorf("imap list folders: %w", err)
+	}
+
+	var folders []FolderInfo
+	for _, mb := range mailboxes {
+		fi := FolderInfo{Name: mb.Mailbox}
+
+		// Check special-use attributes (RFC 6154)
+		for _, attr := range mb.Attrs {
+			if attr == imapv2.MailboxAttrJunk {
+				fi.Excluded = true
+				fi.Reason = "special_use"
+				break
+			}
+			if attr == imapv2.MailboxAttrTrash {
+				fi.Excluded = true
+				fi.Reason = "special_use"
+				break
+			}
+		}
+
+		// Fallback: case-insensitive name matching
+		if !fi.Excluded {
+			lower := strings.ToLower(mb.Mailbox)
+			for _, jt := range junkTrashNames {
+				if lower == jt {
+					fi.Excluded = true
+					fi.Reason = "name_match"
+					break
+				}
+			}
+		}
+
+		folders = append(folders, fi)
+	}
+
+	return folders, nil
+}
@@ -0,0 +1,272 @@
+package imap
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"strings"
+	"time"
+
+	imapv2 "github.com/emersion/go-imap/v2"
+	"github.com/emersion/go-imap/v2/imapclient"
+	"github.com/archivmail/internal/index"
+	"github.com/archivmail/internal/storage"
+	"github.com/archivmail/pkg/mailparser"
+)
+
+const batchSize = 50
+
+// Importer runs background IMAP import jobs.
+type Importer struct {
+	store     *Store
+	mailStore *storage.Store
+	idx       index.Indexer
+	logger    *slog.Logger
+}
+
+// NewImporter creates a new Importer wired to the storage and index backends.
+func NewImporter(store *Store, mailStore *storage.Store, idx index.Indexer, logger *slog.Logger) *Importer {
+	return &Importer{
+		store:     store,
+		mailStore: mailStore,
+		idx:       idx,
+		logger:    logger,
+	}
+}
+
+// Run performs a full IMAP import for the given account. It is designed to be
+// called as a goroutine: go imp.Run(context.Background(), accountID)
+func (imp *Importer) Run(ctx context.Context, accountID int64) {
+	log := imp.logger.With("component", "imap-importer", "account_id", accountID)
+
+	acc, err := imp.store.Get(ctx, accountID)
+	if err != nil {
+		log.Error("failed to get account", "err", err)
+		return
+	}
+
+	password, err := imp.store.GetPassword(ctx, accountID)
+	if err != nil {
+		log.Error("failed to decrypt password", "err", err)
+		_ = imp.store.UpdateStatus(ctx, accountID, "error", "failed to decrypt password", 0, 0)
+		return
+	}
+
+	// Mark as running
+	if err := imp.store.UpdateStatus(ctx, accountID, "running", "", 0, 0); err != nil {
+		log.Error("failed to update status", "err", err)
+		return
+	}
+
+	imported, err := imp.doImport(ctx, acc, password, log)
+	if err != nil {
+		log.Error("import failed", "err", err)
+		_ = imp.store.UpdateStatus(ctx, accountID, "error", err.Error(), 0, 0)
+		return
+	}
+
+	if err := imp.store.UpdateDone(ctx, accountID, imported); err != nil {
+		log.Error("failed to update done", "err", err)
+	}
+
+	log.Info("import completed", "imported", imported)
+}
+
+// doImport handles the actual IMAP connection, folder iteration, and message fetching.
+func (imp *Importer) doImport(ctx context.Context, acc *Account, password string, log *slog.Logger) (int, error) {
+	c, err := Connect(acc.Host, acc.Port, acc.TLS)
+	if err != nil {
+		return 0, fmt.Errorf("connect: %w", err)
+	}
+	defer c.Close()
+
+	// Login
+	if err := c.Login(acc.Username, password).Wait(); err != nil {
+		return 0, fmt.Errorf("login: %w", err)
+	}
+
+	// List all folders
+	folders, err := ListFolders(c)
+	if err != nil {
+		return 0, fmt.Errorf("list folders: %w", err)
+	}
+
+	// Build excluded set from account config
+	excluded := make(map[string]bool)
+	for _, f := range acc.ExcludedFolders {
+		excluded[f] = true
+	}
+
+	// Collect included folders
+	var includedFolders []string
+	for _, f := range folders {
+		if !excluded[f.Name] {
+			includedFolders = append(includedFolders, f.Name)
+		}
+	}
+
+	// Count total messages across all folders first
+	totalMsgs := 0
+	folderUIDs := make(map[string][]imapv2.UID)
+
+	for _, folder := range includedFolders {
+		selectData, err := c.Select(folder, nil).Wait()
+		if err != nil {
+			log.Warn("failed to select folder, skipping", "folder", folder, "err", err)
+			continue
+		}
+		_ = selectData
+
+		searchCmd := c.UIDSearch(&imapv2.SearchCriteria{}, nil)
+		searchData, err := searchCmd.Wait()
+		if err != nil {
+			log.Warn("failed to search folder, skipping", "folder", folder, "err", err)
+			continue
+		}
+
+		uids := searchData.AllUIDs()
+		folderUIDs[folder] = uids
+		totalMsgs += len(uids)
+	}
+
+	log.Info("starting import", "folders", len(includedFolders), "total_messages", totalMsgs)
+	_ = imp.store.UpdateStatus(ctx, acc.ID, "running", "", 0, totalMsgs)
+
+	imported := 0
+	processed := 0
+
+	for _, folder := range includedFolders {
+		uids, ok := folderUIDs[folder]
+		if !ok || len(uids) == 0 {
+			continue
+		}
+
+		// Need to re-select the folder before fetching
+		if _, err := c.Select(folder, nil).Wait(); err != nil {
+			log.Warn("failed to re-select folder", "folder", folder, "err", err)
+			continue
+		}
+
+		log.Info("importing folder", "folder", folder, "messages", len(uids))
+
+		// Process in batches
+		for i := 0; i < len(uids); i += batchSize {
+			end := i + batchSize
+			if end > len(uids) {
+				end = len(uids)
+			}
+			batch := uids[i:end]
+
+			count, err := imp.fetchBatch(ctx, c, batch, log)
+			if err != nil {
+				log.Error("batch fetch error", "folder", folder, "offset", i, "err", err)
+				// Continue with the next batch rather than aborting entirely
+				continue
+			}
+
+			imported += count
+			processed += len(batch)
+
+			_ = imp.store.UpdateStatus(ctx, acc.ID, "running", "", processed, totalMsgs)
+		}
+	}
+
+	return imported, nil
+}
+
+// fetchBatch fetches and stores a batch of messages by UID.
+func (imp *Importer) fetchBatch(ctx context.Context, c *imapclient.Client, uids []imapv2.UID, log *slog.Logger) (int, error) {
+	if len(uids) == 0 {
+		return 0, nil
+	}
+
+	fetchOptions := &imapv2.FetchOptions{
+		UID:         true,
+		BodySection: []*imapv2.FetchItemBodySection{{}},
+	}
+
+	seqSet := imapv2.UIDSetNum(uids...)
+	fetchCmd := c.Fetch(seqSet, fetchOptions)
+
+	imported := 0
+	for {
+		msg := fetchCmd.Next()
+		if msg == nil {
+			break
+		}
+
+		// Collect body sections from this message
+		for {
+			item := msg.Next()
+			if item == nil {
+				break
+			}
+
+			switch body := item.(type) {
+			case imapclient.FetchItemDataBodySection:
+				raw, err := io.ReadAll(body.Literal)
+				if err != nil {
+					log.Warn("failed to read message body", "err", err)
+					continue
+				}
+
+				if err := imp.storeAndIndex(raw, log); err != nil {
+					log.Warn("failed to store/index message", "err", err)
+					continue
+				}
+				imported++
+			}
+		}
+	}
+
+	if err := fetchCmd.Close(); err != nil {
+		return imported, fmt.Errorf("fetch close: %w", err)
+	}
+
+	return imported, nil
+}
+
+// storeAndIndex saves a raw email to storage and indexes it.
+func (imp *Importer) storeAndIndex(raw []byte, log *slog.Logger) error {
+	// Save to file storage (deduplicates by SHA256 automatically)
+	id, err := imp.mailStore.Save(raw, time.Now())
+	if err != nil {
+		return fmt.Errorf("save: %w", err)
+	}
+
+	// Parse for indexing
+	pm, err := mailparser.Parse(raw)
+	if err != nil {
+		log.Warn("failed to parse mail for indexing", "id", id, "err", err)
+		// Store succeeded, just skip indexing for unparseable mails
+		return nil
+	}
+
+	// Build attachment names string
+	var attachNames []string
+	for _, a := range pm.Attachments {
+		if a.Filename != "" {
+			attachNames = append(attachNames, a.Filename)
+		}
+	}
+
+	doc := index.MailDocument{
+		ID:            id,
+		From:          pm.From,
+		To:            strings.Join(pm.To, ", "),
+		Subject:       pm.Subject,
+		Body:          pm.TextBody,
+		AttachNames:   strings.Join(attachNames, " "),
+		HasAttachment: len(pm.Attachments) > 0,
+		Date:          pm.Date,
+		Size:          int64(len(raw)),
+	}
+
+	if err := imp.idx.IndexSync(doc); err != nil {
+		log.Warn("failed to index mail", "id", id, "err", err)
+		// Non-fatal: mail is stored, just not searchable yet
+	}
+
+	return nil
+}
@@ -0,0 +1,259 @@
+package imap
+
+import (
+	"context"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+// Account represents an IMAP account configuration stored in the database.
+type Account struct {
+	ID              int64      `json:"id"`
+	Owner           string     `json:"owner"`
+	Name            string     `json:"name"`
+	Host            string     `json:"host"`
+	Port            int        `json:"port"`
+	TLS             string     `json:"tls"`
+	Username        string     `json:"username"`
+	ExcludedFolders []string   `json:"excluded_folders"`
+	Status          string     `json:"status"`
+	ErrorMsg        string     `json:"error_msg"`
+	LastImportAt    *time.Time `json:"last_import_at,omitempty"`
+	LastImportCount int        `json:"last_import_count"`
+	ProgressCurrent int        `json:"progress_current"`
+	ProgressTotal   int        `json:"progress_total"`
+	CreatedAt       time.Time  `json:"created_at"`
+}
+
+// Store manages IMAP account persistence in PostgreSQL.
+type Store struct {
+	pool   *pgxpool.Pool
+	encKey [32]byte
+}
+
+const createTableSQL = `
+CREATE TABLE IF NOT EXISTS imap_accounts (
+	id              SERIAL PRIMARY KEY,
+	owner           TEXT NOT NULL,
+	name            TEXT NOT NULL,
+	host            TEXT NOT NULL,
+	port            INTEGER NOT NULL DEFAULT 993,
+	tls             TEXT NOT NULL DEFAULT 'ssl',
+	username        TEXT NOT NULL,
+	password_enc    BYTEA NOT NULL,
+	excluded_folders TEXT[] NOT NULL DEFAULT '{}',
+	status          TEXT NOT NULL DEFAULT 'idle',
+	error_msg       TEXT NOT NULL DEFAULT '',
+	last_import_at  TIMESTAMPTZ,
+	last_import_count INTEGER NOT NULL DEFAULT 0,
+	progress_current INTEGER NOT NULL DEFAULT 0,
+	progress_total   INTEGER NOT NULL DEFAULT 0,
+	created_at      TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS idx_imap_accounts_owner ON imap_accounts (owner);
+`
+
+// New creates a new Store, connects to PostgreSQL, and runs the migration.
+func New(dsn, secret string) (*Store, error) {
+	pool, err := pgxpool.New(context.Background(), dsn)
+	if err != nil {
+		return nil, fmt.Errorf("imap store: connect: %w", err)
+	}
+
+	if _, err := pool.Exec(context.Background(), createTableSQL); err != nil {
+		pool.Close()
+		return nil, fmt.Errorf("imap store: migrate: %w", err)
+	}
+
+	key := sha256.Sum256([]byte(secret))
+	return &Store{pool: pool, encKey: key}, nil
+}
+
+// Close releases the database connection pool.
+func (s *Store) Close() {
+	s.pool.Close()
+}
+
+// Create inserts a new IMAP account with an encrypted password.
+func (s *Store) Create(ctx context.Context, acc Account, password string) (*Account, error) {
+	enc, err := encryptPassword(password, s.encKey)
+	if err != nil {
+		return nil, fmt.Errorf("imap store: encrypt password: %w", err)
+	}
+
+	row := s.pool.QueryRow(ctx, `
+		INSERT INTO imap_accounts (owner, name, host, port, tls, username, password_enc, excluded_folders)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+		RETURNING id, created_at`,
+		acc.Owner, acc.Name, acc.Host, acc.Port, acc.TLS, acc.Username, enc, acc.ExcludedFolders,
+	)
+
+	if err := row.Scan(&acc.ID, &acc.CreatedAt); err != nil {
+		return nil, fmt.Errorf("imap store: create: %w", err)
+	}
+
+	acc.Status = "idle"
+	acc.ErrorMsg = ""
+	return &acc, nil
+}
+
+// List returns IMAP accounts. Admins see all accounts; regular users see only their own.
+func (s *Store) List(ctx context.Context, owner string, isAdmin bool) ([]Account, error) {
+	var rows pgx.Rows
+	var err error
+
+	if isAdmin {
+		rows, err = s.pool.Query(ctx, `
+			SELECT id, owner, name, host, port, tls, username, excluded_folders,
+			       status, error_msg, last_import_at, last_import_count,
+			       progress_current, progress_total, created_at
+			FROM imap_accounts ORDER BY id`)
+	} else {
+		rows, err = s.pool.Query(ctx, `
+			SELECT id, owner, name, host, port, tls, username, excluded_folders,
+			       status, error_msg, last_import_at, last_import_count,
+			       progress_current, progress_total, created_at
+			FROM imap_accounts WHERE owner = $1 ORDER BY id`, owner)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("imap store: list: %w", err)
+	}
+	defer rows.Close()
+
+	var accounts []Account
+	for rows.Next() {
+		var a Account
+		if err := rows.Scan(
+			&a.ID, &a.Owner, &a.Name, &a.Host, &a.Port, &a.TLS, &a.Username,
+			&a.ExcludedFolders, &a.Status, &a.ErrorMsg, &a.LastImportAt,
+			&a.LastImportCount, &a.ProgressCurrent, &a.ProgressTotal, &a.CreatedAt,
+		); err != nil {
+			return nil, fmt.Errorf("imap store: scan: %w", err)
+		}
+		accounts = append(accounts, a)
+	}
+	return accounts, rows.Err()
+}
+
+// Get returns a single IMAP account by ID.
+func (s *Store) Get(ctx context.Context, id int64) (*Account, error) {
+	var a Account
+	err := s.pool.QueryRow(ctx, `
+		SELECT id, owner, name, host, port, tls, username, excluded_folders,
+		       status, error_msg, last_import_at, last_import_count,
+		       progress_current, progress_total, created_at
+		FROM imap_accounts WHERE id = $1`, id,
+	).Scan(
+		&a.ID, &a.Owner, &a.Name, &a.Host, &a.Port, &a.TLS, &a.Username,
+		&a.ExcludedFolders, &a.Status, &a.ErrorMsg, &a.LastImportAt,
+		&a.LastImportCount, &a.ProgressCurrent, &a.ProgressTotal, &a.CreatedAt,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("imap store: get %d: %w", id, err)
+	}
+	return &a, nil
+}
+
+// GetPassword retrieves and decrypts the stored password for an IMAP account.
+func (s *Store) GetPassword(ctx context.Context, id int64) (string, error) {
+	var enc []byte
+	err := s.pool.QueryRow(ctx, `SELECT password_enc FROM imap_accounts WHERE id = $1`, id).Scan(&enc)
+	if err != nil {
+		return "", fmt.Errorf("imap store: get password: %w", err)
+	}
+	return decryptPassword(enc, s.encKey)
+}
+
+// Delete removes an IMAP account by ID.
+func (s *Store) Delete(ctx context.Context, id int64) error {
+	tag, err := s.pool.Exec(ctx, `DELETE FROM imap_accounts WHERE id = $1`, id)
+	if err != nil {
+		return fmt.Errorf("imap store: delete: %w", err)
+	}
+	if tag.RowsAffected() == 0 {
+		return fmt.Errorf("imap store: account %d not found", id)
+	}
+	return nil
+}
+
+// UpdateExcluded sets the list of excluded folders for an account.
+func (s *Store) UpdateExcluded(ctx context.Context, id int64, excluded []string) error {
+	_, err := s.pool.Exec(ctx, `UPDATE imap_accounts SET excluded_folders = $1 WHERE id = $2`, excluded, id)
+	if err != nil {
+		return fmt.Errorf("imap store: update excluded: %w", err)
+	}
+	return nil
+}
+
+// UpdateStatus updates the import progress and status of an account.
+func (s *Store) UpdateStatus(ctx context.Context, id int64, status, errMsg string, current, total int) error {
+	_, err := s.pool.Exec(ctx, `
+		UPDATE imap_accounts
+		SET status = $1, error_msg = $2, progress_current = $3, progress_total = $4
+		WHERE id = $5`, status, errMsg, current, total, id)
+	if err != nil {
+		return fmt.Errorf("imap store: update status: %w", err)
+	}
+	return nil
+}
+
+// UpdateDone marks an import as completed, setting status back to idle.
+func (s *Store) UpdateDone(ctx context.Context, id int64, count int) error {
+	_, err := s.pool.Exec(ctx, `
+		UPDATE imap_accounts
+		SET status = 'idle', error_msg = '', last_import_at = now(),
+		    last_import_count = $1, progress_current = 0, progress_total = 0
+		WHERE id = $2`, count, id)
+	if err != nil {
+		return fmt.Errorf("imap store: update done: %w", err)
+	}
+	return nil
+}
+
+// encryptPassword encrypts a plaintext password using AES-256-GCM.
+func encryptPassword(plaintext string, key [32]byte) ([]byte, error) {
+	block, err := aes.NewCipher(key[:])
+	if err != nil {
+		return nil, err
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+	nonce := make([]byte, gcm.NonceSize())
+	if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+		return nil, err
+	}
+	return gcm.Seal(nonce, nonce, []byte(plaintext), nil), nil
+}
+
+// decryptPassword decrypts a password previously encrypted with encryptPassword.
+func decryptPassword(ciphertext []byte, key [32]byte) (string, error) {
+	block, err := aes.NewCipher(key[:])
+	if err != nil {
+		return "", err
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return "", err
+	}
+	nonceSize := gcm.NonceSize()
+	if len(ciphertext) < nonceSize {
+		return "", fmt.Errorf("ciphertext too short")
+	}
+	nonce, ct := ciphertext[:nonceSize], ciphertext[nonceSize:]
+	plaintext, err := gcm.Open(nil, nonce, ct, nil)
+	if err != nil {
+		return "", fmt.Errorf("decrypt failed: %w", err)
+	}
+	return string(plaintext), nil
+}