From 4eb300c2529225a8d63305ff258ed50df9ecb7c3 Mon Sep 17 00:00:00 2001 From: patrick Date: Thu, 29 Oct 2026 14:38:27 +0100 Subject: [PATCH] feat(PROJ-94): Postfach-Zuordnung pro IMAP-Konto, imap-link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tabelle imap_account_mails, Importer schreibt die Zuordnung auch bei Duplikaten, Zugriffsregel und Index-Tokens nutzen sie, archivmail imap-link befüllt den Bestand (Trockenlauf Standard, --apply schreibt nur die Zuordnung). Co-Authored-By: Claude Sonnet 5.5 --- cmd/archivmail/cmd_imap_link.go | 125 +++++++++++ cmd/archivmail/cmd_import.go | 9 + cmd/archivmail/main.go | 3 + features/INDEX.md | 3 +- .../PROJ-94-postfach-zuordnung-pro-konto.md | 58 +++++ internal/api/search_handlers.go | 4 +- internal/api/user_account_access.go | 33 ++- internal/api/user_account_links_test.go | 72 ++++++ internal/imap/account_mails.go | 65 ++++++ internal/imap/account_mails_test.go | 209 ++++++++++++++++++ internal/imap/importer.go | 6 + internal/imap/link.go | 189 ++++++++++++++++ internal/imap/link_output.go | 56 +++++ internal/imap/link_source.go | 48 ++++ internal/imap/link_test.go | 146 ++++++++++++ internal/imap/store.go | 29 ++- internal/storage/delivery_rcpt.go | 9 +- internal/storage/mail_links.go | 39 ++++ internal/storage/mail_source.go | 26 +++ internal/storage/mail_source_test.go | 31 +++ internal/storage/storage.go | 12 + src/data/features.ts | 1 + 22 files changed, 1152 insertions(+), 21 deletions(-) create mode 100644 cmd/archivmail/cmd_imap_link.go create mode 100644 features/PROJ-94-postfach-zuordnung-pro-konto.md create mode 100644 internal/api/user_account_links_test.go create mode 100644 internal/imap/account_mails.go create mode 100644 internal/imap/account_mails_test.go create mode 100644 internal/imap/link.go create mode 100644 internal/imap/link_output.go create mode 100644 internal/imap/link_source.go create mode 100644 internal/imap/link_test.go create mode 100644 internal/storage/mail_links.go diff --git a/cmd/archivmail/cmd_imap_link.go b/cmd/archivmail/cmd_imap_link.go new file mode 100644 index 0000000..5c30af3 --- /dev/null +++ b/cmd/archivmail/cmd_imap_link.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + + "github.com/jackc/pgx/v5/pgxpool" + + "archivmail/config" + imapstore "archivmail/internal/imap" +) + +func runIMAPLink(args []string) { + fs := flag.NewFlagSet("imap-link", flag.ExitOnError) + configPath := fs.String("config", "/etc/archivmail/config.yml", "path to config file") + accountID := fs.Int64("account", 0, "IMAP account ID (0 = all accounts)") + folder := fs.String("folder", "", "check only this folder") + apply := fs.Bool("apply", false, "write missing links (default: dry run)") + jsonOut := fs.Bool("json", false, "machine-readable JSON output") + fs.Parse(args) + + cfg, err := config.Load(*configPath) + if err != nil { + fmt.Fprintf(os.Stderr, "imap-link: load config: %v\n", err) + os.Exit(2) + } + + aesKey, err := deriveIMAPKey(cfg.API.Secret) + if err != nil { + fmt.Fprintf(os.Stderr, "imap-link: key derivation failed\n") + os.Exit(2) + } + + ctx := context.Background() + roPool, err := openLinkPool(ctx, cfg.Database.DSN(), true) + if err != nil { + fmt.Fprintf(os.Stderr, "imap-link: %v\n", err) + os.Exit(2) + } + defer roPool.Close() + + // Only the link INSERT goes through a writable pool; everything else + // stays on the server-enforced read-only pool, as in imap-verify. + var write imapstore.LinkWriter + if *apply { + rwPool, err := openLinkPool(ctx, cfg.Database.DSN(), false) + if err != nil { + fmt.Fprintf(os.Stderr, "imap-link: %v\n", err) + os.Exit(2) + } + defer rwPool.Close() + write = imapstore.NewLinkWriter(rwPool) + } + + st := imapstore.NewReadOnlyStore(roPool, aesKey) + var accounts []imapstore.Account + if *accountID > 0 { + acc, err := st.Get(ctx, *accountID) + if err != nil { + fmt.Fprintf(os.Stderr, "imap-link: account %d not found\n", *accountID) + os.Exit(2) + } + accounts = []imapstore.Account{*acc} + } else { + accounts, err = st.ListAll(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "imap-link: list accounts: %v\n", err) + os.Exit(2) + } + } + + lookup := imapstore.NewLinkLookup(roPool) + reports := make([]imapstore.LinkAccountReport, 0, len(accounts)) + for i := range accounts { + reports = append(reports, linkIMAPAccount(ctx, st, &accounts[i], lookup, write, *folder)) + } + + code := imapstore.LinkExitCode(reports) + if *jsonOut { + if err := imapstore.WriteLinkJSON(os.Stdout, reports, *apply); err != nil { + fmt.Fprintf(os.Stderr, "imap-link: write json: %v\n", err) + code = 2 + } + } else { + imapstore.WriteLinkText(os.Stdout, reports, *apply) + } + roPool.Close() + os.Exit(code) +} + +func openLinkPool(ctx context.Context, dsn string, readOnly bool) (*pgxpool.Pool, error) { + pcfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, fmt.Errorf("invalid database config") + } + if readOnly { + pcfg.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on" + pcfg.MaxConns = 2 + } else { + pcfg.MaxConns = 1 + } + pool, err := pgxpool.NewWithConfig(ctx, pcfg) + if err != nil { + return nil, fmt.Errorf("database connect: %w", err) + } + return pool, nil +} + +func linkIMAPAccount(ctx context.Context, st *imapstore.Store, acc *imapstore.Account, lookup imapstore.LinkLookup, write imapstore.LinkWriter, folder string) imapstore.LinkAccountReport { + password, err := st.GetPassword(ctx, acc.ID) + if err != nil { + return imapstore.FailedLinkReport(acc.ID, fmt.Errorf("decrypt password failed"), nil) + } + secrets := []string{password, acc.Username} + + src, closeFn, err := imapstore.DialVerifySource(acc, password) + if err != nil { + return imapstore.FailedLinkReport(acc.ID, err, secrets) + } + defer closeFn() + + return imapstore.LinkAccount(ctx, acc.ID, src, lookup, write, acc.ExcludedFolders, folder, secrets) +} diff --git a/cmd/archivmail/cmd_import.go b/cmd/archivmail/cmd_import.go index 6a29a5d..293f518 100644 --- a/cmd/archivmail/cmd_import.go +++ b/cmd/archivmail/cmd_import.go @@ -319,6 +319,7 @@ Commands: update Auf neueste Version aktualisieren (führt update.sh aus) status Healthcheck für DB, Manticore und Storage imap-verify IMAP-Postfächer read-only gegen das Archiv abgleichen (PROJ-93) + imap-link Postfach-Zuordnung pro IMAP-Konto nachtragen (PROJ-94, Trockenlauf per Default) version Version anzeigen help Diese Hilfe anzeigen @@ -377,5 +378,13 @@ archivmail imap-verify [flags] --folder Nur diesen Ordner prüfen --json Maschinenlesbare JSON-Ausgabe Exit-Code: 0 vollständig, 1 fehlende/nicht prüfbare Mails, 2 Fehler + +archivmail imap-link [flags] + --config Pfad zur Konfigurationsdatei (Standard: /etc/archivmail/config.yml) + --account IMAP-Konto-ID (0 = alle Konten) + --folder Nur diesen Ordner prüfen + --apply Fehlende Zuordnungen schreiben (ohne: Trockenlauf) + --json Maschinenlesbare JSON-Ausgabe + Exit-Code: 0 ok, 2 Fehler `, AppVersion) } diff --git a/cmd/archivmail/main.go b/cmd/archivmail/main.go index 4ce9872..709fdea 100644 --- a/cmd/archivmail/main.go +++ b/cmd/archivmail/main.go @@ -93,6 +93,9 @@ func main() { case "imap-verify": runIMAPVerify(os.Args[2:]) return + case "imap-link": + runIMAPLink(os.Args[2:]) + return case "update": runUpdate(os.Args[2:]) return diff --git a/features/INDEX.md b/features/INDEX.md index 4f719b1..3c1677c 100644 --- a/features/INDEX.md +++ b/features/INDEX.md @@ -109,7 +109,8 @@ | PROJ-91 | Envelope-Absender (Return-Path) durchsuchbar machen | In Review | [PROJ-91](PROJ-91-envelope-absender-suche.md) | 2026-10-29 | | PROJ-92 | Weiterleitung in Liste und Detail anzeigen | In Review | [PROJ-92](PROJ-92-weiterleitung-anzeigen.md) | 2026-10-29 | | PROJ-93 | IMAP-Vollständigkeitsprüfung (Postfach vs. Archiv) | In Review | [PROJ-93](PROJ-93-imap-vollstaendigkeitspruefung.md) | 2026-10-29 | +| PROJ-94 | Postfach-Zuordnung pro IMAP-Konto (Archiv spiegelt Posteingang) | In Progress | [PROJ-94](PROJ-94-postfach-zuordnung-pro-konto.md) | 2026-10-29 | -## Next Available ID: PROJ-94 +## Next Available ID: PROJ-95 diff --git a/features/PROJ-94-postfach-zuordnung-pro-konto.md b/features/PROJ-94-postfach-zuordnung-pro-konto.md new file mode 100644 index 0000000..26a11a3 --- /dev/null +++ b/features/PROJ-94-postfach-zuordnung-pro-konto.md @@ -0,0 +1,58 @@ +# PROJ-94: Postfach-Zuordnung pro IMAP-Konto (Archiv spiegelt Posteingang) + +## Status: In Progress +**Created:** 2026-10-29 +**Last Updated:** 2026-10-29 + +## Dependencies +- Requires: PROJ-90 (Konto-Owner-Sichtbarkeit), PROJ-93 (imap-verify), PROJ-52 (Quelle `source_type/source_id`) +- Ersetzt die Grenze aus PROJ-90: "Quelle des ersten Eingangs entscheidet" + +## Hintergrund +PROJ-90 macht Mails über `emails.source_type='imap'` + `source_id` sichtbar. Diese Quelle wird nur beim ersten Speichern gesetzt (`TagSource` first-write-wins). Mails, die im Postfach liegen, aber zuerst über einen anderen Weg (Import, anderes Konto) archiviert wurden, bleiben unsichtbar. Beobachtet: 17 Altmails (24.04.–02.07.2026, `source_type NULL`) im Postfach von `stefan@perlbach24.de`, laut `imap-verify` im Archiv vorhanden, für den Owner unsichtbar. Das Archiv soll den Posteingang widerspiegeln. + +## User Stories +- Als User möchte ich jede Mail sehen, die in meinem IMAP-Postfach liegt, unabhängig davon, auf welchem Weg sie zuerst archiviert wurde. + +## Acceptance Criteria +- [ ] Neue Tabelle `imap_account_mails (account_id, mail_id, first_seen_at)`, PK `(account_id, mail_id)`, FK auf `imap_accounts(id) ON DELETE CASCADE`, Index auf `mail_id`. Anlage idempotent in `initSchema`/Migration (Muster der vorhandenen Migrationen). Die Herkunftsangabe `emails.source_type/source_id` bleibt unverändert und wird nie von dieser Funktion überschrieben. +- [ ] Der IMAP-Importer (alle Abrufpfade: Erstimport, Sync/Scheduler) legt nach `Save` die Zuordnung `(account_id, mail_id)` per `INSERT … ON CONFLICT DO NOTHING` an, auch wenn die Mail bereits existiert (Message-ID- oder SHA-256-Dedup). Fehler dabei sind nicht fatal (Warn-Log), die Mail bleibt gespeichert. +- [ ] Sichtbarkeitsregel Rolle `user`: Mail sichtbar, wenn für ein dem User gehörendes Konto (Owner = Username oder E-Mail, Tenant gleich, wie PROJ-90) eine Zuordnung existiert ODER die bisherigen Regeln (PROJ-90 Quelle, Adressen) greifen. Detailzugriff prüft per DB und ist maßgeblich. +- [ ] Index: `DeliveryRcpt` hängt für jede Mail die Tokens `imapacct` ALLER Konten mit Zuordnung an (ein Token pro Konto); bisheriges Verhalten (Token der Quelle) bleibt als Teilmenge erhalten. Suche/Token-Validierung unverändert. +- [ ] Neuer CLI-Befehl `archivmail imap-link [--config …] [--account ] [--folder ] [--apply]`: gleicht das Postfach wie `imap-verify` ab (nur lesend gegen den IMAP-Server, EXAMINE + BODY.PEEK) und schreibt mit `--apply` nur fehlende Zuordnungen für Mails, deren Message-ID im Archiv existiert. Ohne `--apply` Trockenlauf: nur Zahlen (zugeordnet neu / bereits vorhanden / nicht im Archiv). Ausgabe ohne Adressen, Betreffs, Message-IDs, Passwörter. +- [ ] Mandantengrenze: Zuordnung nur für Mails, die zum Tenant des Kontos gehören bzw. im selben Tenant sichtbar sind (`emails.tenant_id` = `imap_accounts.tenant_id` oder die Mail ist über `email_refs` diesem Tenant zugeordnet). Andernfalls nicht zuordnen und als "übersprungen (Tenant)" zählen. +- [ ] Konto gelöscht: Zuordnungen verschwinden per CASCADE; kein Zugriff mehr darüber. +- [ ] Bestand: nach Deploy `archivmail imap-link --apply`, danach `archivmail reindex`, damit die Liste die Tokens kennt. Detailzugriff wirkt nach `imap-link` sofort. +- [ ] Tests (synthetisch): Migration idempotent, Zuordnung bei Duplikat-Mail, Sichtbarkeit mit Zuordnung ohne Quelle, Fremdkonto/falscher Tenant, Konto gelöscht, Token-Bau mit mehreren Konten, Trockenlauf schreibt nicht, Ausgabe ohne Geheimnisse. + +## Edge Cases +- Dieselbe Mail in mehreren Konten desselben oder verschiedener Owner: je Konto eine Zeile; jeder Owner sieht sie. +- Mail ohne Message-ID im Archiv: wird von `imap-link` nicht zugeordnet, sondern gezählt. +- Spam/Trash (excluded_folders): nicht geprüft, nicht zugeordnet. +- Löschen im Postfach entfernt die Zuordnung NICHT (GoBD: Archiv bleibt). + +## Technical Requirements +- Security: Zuordnung entsteht nur durch den Importer bzw. `imap-link` mit Server-Beleg, nie aus Mail-Headern; Tenant-Isolation; Fail-closed bei DB-Fehlern. +- GoBD: Herkunftsangabe unverändert; neue Tabelle nur Zuordnung. +- Deployment: update.sh (Migration beim Start), dann `imap-link --apply` (Freigabe des Users), dann `archivmail reindex` (Freigabe des Users). + +## Implementation Notes +**Backend (2026-10-29):** +- Schema: `internal/imap/account_mails.go` (`accountMailsSQL`), ausgeführt in `migrate()` in `internal/imap/store.go` (die bisherigen drei Migrationsschritte wurden dorthin verschoben, Verhalten von `New()` unverändert). `imap_account_mails(account_id BIGINT FK imap_accounts ON DELETE CASCADE, mail_id TEXT, first_seen_at)`, PK `(account_id, mail_id)`, Index `idx_imap_account_mails_mail`. Kein FK auf `emails` (Schema gehört zu `internal/storage`); stattdessen löscht `storage.Delete` die Zuordnungen in derselben Transaktion (nur wenn die Tabelle existiert, `to_regclass`). +- Tenant-Regel als SQL-Guard im INSERT (`linkAccountMailSQL`): `emails.tenant_id IS NOT DISTINCT FROM imap_accounts.tenant_id` ODER `email_refs` für den Konto-Tenant. Gilt für Importer und `imap-link` gleichermaßen (Routing-Regeln PROJ-43 können eine Mail unter einem anderen Tenant speichern; die wird dann nicht zugeordnet). Konto ohne Tenant: nur Mails ohne Tenant. +- Importer: `storeAndIndex` ruft nach `Save`/`TagSource` `Store.LinkMail` (ON CONFLICT DO NOTHING, auch bei Duplikaten), Fehler nur Warn-Log. Vor `DeliveryRcpt`, damit der Index-Eintrag das Token sofort trägt. Alle IMAP-Abrufpfade (Erstimport `Run`/`fetchBatch`, Scheduler-Sync) laufen über `storeAndIndex`; POP3 unberührt. `TagSource` unverändert. +- Lesen: `internal/storage/mail_links.go` (`LinkedImapAccounts`, `LinkedImapAccountsBatch`). `MailSource.Linked` + `AccountIDs()`/`AccountTokens()` (Quelle ∪ Zuordnungen, dedupliziert, aufsteigend sortiert). +- Zugriff (`internal/api/user_account_access.go`): `allows` prüft alle `AccountIDs()` gegen die eigenen Konten. `userOwnsMail` lädt Zuordnungen separat; DB-Fehler → keine Zuordnungen (Warn-Log), bestehende Regeln greifen weiter. Ein Fehler der Quell-Abfrage verwirft weiterhin Quelle und Zustell-Header-Zweig, die separat geladenen Zuordnungen bleiben. Listen-Nachprüfung (`search_handlers.go`) lädt Zuordnungen per Batch (`linkedAccountsBatch`, `withLinks`). +- Index: `DeliveryRcpt` hängt `AccountTokens()` an; Link-Lookup-Fehler → nur Quell-Token (Detailprüfung liest ohnehin per DB). +- CLI `cmd/archivmail/cmd_imap_link.go`, Logik `internal/imap/link.go`, DB-Lookup `internal/imap/link_source.go`, Ausgabe `internal/imap/link_output.go`. Wiederverwendet `DialVerifySource` (EXAMINE, `UID SEARCH ALL`, `BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)]`) und `NormalizeMessageID`. Lese-Pool mit `default_transaction_read_only=on`; nur mit `--apply` ein zweiter Pool (1 Verbindung), der ausschließlich den Link-INSERT ausführt. Keine Migrationen, kein Store/Manticore, kein Audit-Eintrag (wäre ein weiterer DB-Write außerhalb `imap_account_mails`). +- Zählung pro Mail im Postfach in genau einer Kategorie: neu, bereits zugeordnet (inkl. zweites Vorkommen derselben Mail in einem anderen Ordner des Kontos bzw. Konflikt beim INSERT), nicht im Archiv, ohne Message-ID, übersprungen (Tenant). Trockenlauf und `--apply` liefern dieselben Zahlen. Ausgeschlossene/`\Noselect`-Ordner werden nicht abgerufen. Exit-Code 0 ok, 2 Fehler. +- Ausgabe: nur Konto-ID, Ordnername, Zähler; Fehlertexte über `RedactSecrets` (Passwort, IMAP-Benutzername). +- Tests: `internal/imap/link_test.go` (Zählung, Trockenlauf = Apply-Zahlen, Writer nur bei `--apply` und nur für neue, Fehler, Ausschluss, keine Geheimnisse/Message-IDs in Text/JSON), `internal/imap/account_mails_test.go` (SQL-Guard per Fake; Integration mit `TEST_DATABASE_URL`: Migration 2× idempotent, Duplikat, Tenant via `emails.tenant_id`/`email_refs`, falscher Tenant, Konto ohne Tenant, Quelle unverändert, CASCADE, Trockenlauf schreibt nicht), `internal/storage/mail_source_test.go` (Token-Bau mehrere Konten), `internal/api/user_account_links_test.go` (Sichtbarkeit ohne Quelle, Fremdkonto, falscher Tenant, Konto gelöscht, Fail-closed, Liste = Detail). `go vet ./...`, `go test ./internal/... ./pkg/...`, `go build ./cmd/...` auf 132 im Scratch-Verzeichnis grün (danach gelöscht). Die DB-Integrationstests liefen dort mangels `TEST_DATABASE_URL` als SKIP. + +**Aufruf:** `archivmail imap-link [--config /etc/archivmail/config.yml] [--account ] [--folder ] [--apply] [--json]` + +**Bekannte Grenzen:** +- `imap-link` setzt die Migration voraus (erst nach Deploy/Neustart des Daemons sinnvoll); vorher schlägt der Lookup fehl (`failed`). +- Mails ohne `message_id` in `emails` (beim Import nicht parsebar) werden von `imap-link` als „nicht im Archiv“ gezählt; der Importer ordnet sie bei künftigen Abrufen trotzdem zu. +- Liste braucht nach `imap-link --apply` ein `archivmail reindex`; Detailzugriff wirkt sofort. +- Noch nicht gegen ein echtes Postfach ausgeführt. diff --git a/internal/api/search_handlers.go b/internal/api/search_handlers.go index 57c3f50..1b4561a 100644 --- a/internal/api/search_handlers.go +++ b/internal/api/search_handlers.go @@ -191,12 +191,14 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { // contains the phrase of alice@example.com), so user hits are re-checked // with the exact detail-access rule before any metadata is returned. var userSources map[string]storage.MailSource + var userLinks map[string][]int64 if access != nil { var srcErr error userSources, srcErr = s.store.GetMailSources(r.Context(), hitIDs) if srcErr != nil { userSources = map[string]storage.MailSource{} } + userLinks = s.linkedAccountsBatch(r.Context(), hitIDs) } dropped := 0 @@ -209,7 +211,7 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { if loadErr == nil { pm, parseErr = mailparser.Parse(raw) } - if access != nil && (parseErr != nil || !access.allows(pm, userSources[h.ID])) { + if access != nil && (parseErr != nil || !access.allows(pm, withLinks(userSources[h.ID], userLinks[h.ID]))) { dropped++ continue } diff --git a/internal/api/user_account_access.go b/internal/api/user_account_access.go index 52bf2d8..73015a5 100644 --- a/internal/api/user_account_access.go +++ b/internal/api/user_account_access.go @@ -64,7 +64,7 @@ func (a *mailAccess) allows(pm *mailparser.ParsedMail, src storage.MailSource) b if a == nil { return false } - if id, ok := src.ImapAccountID(); ok { + for _, id := range src.AccountIDs() { if _, own := a.accounts[id]; own { return true } @@ -103,16 +103,41 @@ func accountsOrNone(ids []int64, err error) []int64 { return ids } -// A failed source lookup drops both the delivery-header and the account path; -// From/To/CC/BCC still apply, so the result is never more permissive. +// A failed source lookup drops both the delivery-header and the source +// account path; From/To/CC/BCC and the separately loaded links still apply, +// so the result is never more permissive. func (s *Server) userOwnsMail(ctx context.Context, id string, pm *mailparser.ParsedMail, acc *mailAccess) bool { src, err := s.store.GetMailSource(ctx, id) + src.Linked = s.linkedAccounts(ctx, id) return mailAccessDecision(pm, src, err, acc) } func mailAccessDecision(pm *mailparser.ParsedMail, src storage.MailSource, srcErr error, acc *mailAccess) bool { if srcErr != nil { - src = storage.MailSource{} + src = storage.MailSource{Linked: src.Linked} } return acc.allows(pm, src) } + +func (s *Server) linkedAccounts(ctx context.Context, id string) []int64 { + linked, err := s.store.LinkedImapAccounts(ctx, id) + if err != nil { + s.logger.Warn("imap account link lookup failed, using source rules only", "err", err) + return nil + } + return linked +} + +func (s *Server) linkedAccountsBatch(ctx context.Context, ids []string) map[string][]int64 { + linked, err := s.store.LinkedImapAccountsBatch(ctx, ids) + if err != nil { + s.logger.Warn("imap account link lookup failed, using source rules only", "err", err) + return map[string][]int64{} + } + return linked +} + +func withLinks(src storage.MailSource, linked []int64) storage.MailSource { + src.Linked = linked + return src +} diff --git a/internal/api/user_account_links_test.go b/internal/api/user_account_links_test.go new file mode 100644 index 0000000..8220c98 --- /dev/null +++ b/internal/api/user_account_links_test.go @@ -0,0 +1,72 @@ +package api + +import ( + "errors" + "slices" + "testing" + + "archivmail/internal/storage" +) + +func TestMailAccessLinkedAccountWithoutSource(t *testing.T) { + pm := parseTestMail(t, "From: sender@example.org\r\nTo: forward-target@example.net\r\n") + acc := newMailAccess([]string{"login@example.com"}, []int64{3}) + tests := []struct { + name string + src storage.MailSource + want bool + }{ + {"legacy mail linked to own account", storage.MailSource{Linked: []int64{3}}, true}, + {"smtp source, linked to own account", storage.MailSource{Type: "smtp", Linked: []int64{3}}, true}, + {"foreign account source, also linked to own", storage.MailSource{Type: "imap", ID: ptrInt64(9), Linked: []int64{9, 3}}, true}, + {"linked only to foreign account", storage.MailSource{Linked: []int64{4}}, false}, + {"linked to account of another tenant", storage.MailSource{Linked: []int64{5}}, false}, + {"account deleted, links cascaded", storage.MailSource{}, false}, + } + for _, tt := range tests { + if got := acc.allows(pm, tt.src); got != tt.want { + t.Errorf("%s: got %v, want %v", tt.name, got, tt.want) + } + } +} + +func TestMailAccessSourceErrorKeepsLinksOnly(t *testing.T) { + pm := parseTestMail(t, "Delivered-To: login@example.com\r\nFrom: a@example.org\r\nTo: b@example.org\r\n") + owner := newMailAccess([]string{"login@example.com"}, []int64{3}) + if !mailAccessDecision(pm, storage.MailSource{Linked: []int64{3}}, errors.New("db down"), owner) { + t.Error("links are loaded separately and must survive a source lookup error") + } + if mailAccessDecision(pm, storage.MailSource{Type: "imap", ID: ptrInt64(3)}, errors.New("db down"), owner) { + t.Error("source lookup error must still drop the source account path") + } + if mailAccessDecision(pm, storage.MailSource{Type: "imap", Linked: nil}, errors.New("db down"), newMailAccess([]string{"login@example.com"}, nil)) { + t.Error("source lookup error must still drop the delivery-header path") + } +} + +func TestAccountLinkListDetailConsistency(t *testing.T) { + pm := parseTestMail(t, "From: x@example.org\r\nTo: y@example.org\r\n") + acc := newMailAccess([]string{"login@example.com"}, []int64{3}) + tokens := acc.accountTokens() + for _, src := range []storage.MailSource{ + {Linked: []int64{3}}, + {Linked: []int64{4, 7}}, + {Type: "imap", ID: ptrInt64(4), Linked: []int64{3}}, + {Type: "smtp"}, + } { + inList := false + for _, tok := range src.AccountTokens() { + if slices.Contains(tokens, tok) { + inList = true + } + } + if got := mailAccessDecision(pm, src, nil, acc); got != inList { + t.Errorf("%+v: detail=%v, list=%v", src, got, inList) + } + if got := acc.allows(pm, withLinks(storage.MailSource{Type: src.Type, ID: src.ID}, src.Linked)); got != inList { + t.Errorf("%+v: list re-check=%v, index=%v", src, got, inList) + } + } +} + +func ptrInt64(v int64) *int64 { return &v } diff --git a/internal/imap/account_mails.go b/internal/imap/account_mails.go new file mode 100644 index 0000000..64f772a --- /dev/null +++ b/internal/imap/account_mails.go @@ -0,0 +1,65 @@ +package imap + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5/pgconn" +) + +// PROJ-94: which archived mails were seen in which IMAP account. Separate from +// emails.source_type/source_id, which stays first-write-wins (GoBD). +const accountMailsSQL = ` +CREATE TABLE IF NOT EXISTS imap_account_mails ( + account_id BIGINT NOT NULL REFERENCES imap_accounts(id) ON DELETE CASCADE, + mail_id TEXT NOT NULL, + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (account_id, mail_id) +); +CREATE INDEX IF NOT EXISTS idx_imap_account_mails_mail ON imap_account_mails (mail_id); +` + +// The tenant guard lives in SQL so importer and imap-link enforce the same +// boundary: a routing rule may have stored the mail under another tenant than +// the account's, and that mail must not become visible to the account owner. +const linkAccountMailSQL = ` +INSERT INTO imap_account_mails (account_id, mail_id) +SELECT a.id, e.id +FROM imap_accounts a +JOIN emails e ON e.id = $2 +WHERE a.id = $1 + AND (e.tenant_id IS NOT DISTINCT FROM a.tenant_id + OR (a.tenant_id IS NOT NULL AND EXISTS ( + SELECT 1 FROM email_refs r WHERE r.email_id = e.id AND r.tenant_id = a.tenant_id))) +ON CONFLICT (account_id, mail_id) DO NOTHING +` + +type execer interface { + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) +} + +type LinkWriter func(ctx context.Context, accountID int64, mailID string) (bool, error) + +func NewLinkWriter(db execer) LinkWriter { + return func(ctx context.Context, accountID int64, mailID string) (bool, error) { + return linkAccountMail(ctx, db, accountID, mailID) + } +} + +func linkAccountMail(ctx context.Context, db execer, accountID int64, mailID string) (bool, error) { + if db == nil || accountID <= 0 || mailID == "" { + return false, nil + } + tag, err := db.Exec(ctx, linkAccountMailSQL, accountID, mailID) + if err != nil { + return false, fmt.Errorf("imap: link account mail: %w", err) + } + return tag.RowsAffected() > 0, nil +} + +func (s *Store) LinkMail(ctx context.Context, accountID int64, mailID string) (bool, error) { + if s == nil || s.pool == nil { + return false, nil + } + return linkAccountMail(ctx, s.pool, accountID, mailID) +} diff --git a/internal/imap/account_mails_test.go b/internal/imap/account_mails_test.go new file mode 100644 index 0000000..850999f --- /dev/null +++ b/internal/imap/account_mails_test.go @@ -0,0 +1,209 @@ +package imap + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +type captureExec struct { + sql string + args []any + rows int64 +} + +func (c *captureExec) Exec(_ context.Context, sql string, args ...any) (pgconn.CommandTag, error) { + c.sql, c.args = sql, args + return pgconn.NewCommandTag(fmt.Sprintf("INSERT 0 %d", c.rows)), nil +} + +func TestLinkAccountMailSQL(t *testing.T) { + c := &captureExec{rows: 1} + ok, err := linkAccountMail(context.Background(), c, 3, "abc") + if err != nil || !ok { + t.Fatalf("got (%v, %v)", ok, err) + } + for _, want := range []string{"ON CONFLICT (account_id, mail_id) DO NOTHING", "a.tenant_id", "email_refs"} { + if !strings.Contains(c.sql, want) { + t.Errorf("link SQL lacks %q", want) + } + } + if strings.Contains(c.sql, "source_type") || strings.Contains(c.sql, "UPDATE") { + t.Error("link must never touch the stored source") + } + + c = &captureExec{rows: 0} + if ok, _ := linkAccountMail(context.Background(), c, 3, "abc"); ok { + t.Error("existing link must report not inserted") + } + for _, tc := range []struct { + acc int64 + id string + }{{0, "abc"}, {3, ""}} { + c = &captureExec{} + if ok, err := linkAccountMail(context.Background(), c, tc.acc, tc.id); ok || err != nil || c.sql != "" { + t.Errorf("%+v: invalid input must be a no-op", tc) + } + } + var nilStore *Store + if ok, err := nilStore.LinkMail(context.Background(), 1, "x"); ok || err != nil { + t.Error("nil store must be a no-op") + } +} + +func testSchemaPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("TEST_DATABASE_URL") + if dsn == "" { + t.Skip("TEST_DATABASE_URL not set — skipping (needs PostgreSQL)") + } + ctx := context.Background() + schema := fmt.Sprintf("proj94_test_%d", time.Now().UnixNano()) + admin, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil { + admin.Close() + t.Fatal(err) + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.RuntimeParams["search_path"] = schema + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + pool.Close() + _, _ = admin.Exec(ctx, "DROP SCHEMA "+schema+" CASCADE") + admin.Close() + }) + if _, err := pool.Exec(ctx, ` + CREATE TABLE tenants (id SERIAL PRIMARY KEY); + CREATE TABLE emails (id TEXT PRIMARY KEY, message_id TEXT, tenant_id BIGINT, source_type TEXT, source_id BIGINT); + CREATE TABLE email_refs (email_id TEXT NOT NULL, tenant_id BIGINT NOT NULL, UNIQUE(email_id, tenant_id)); + INSERT INTO tenants DEFAULT VALUES; INSERT INTO tenants DEFAULT VALUES; + `); err != nil { + t.Fatal(err) + } + return pool +} + +func TestAccountMailsIntegration(t *testing.T) { + pool := testSchemaPool(t) + ctx := context.Background() + + for i := 0; i < 2; i++ { + if err := migrate(ctx, pool); err != nil { + t.Fatalf("migration run %d: %v", i+1, err) + } + } + + var acc1, acc2, accNoTenant int64 + mk := func(owner string, tenant any) int64 { + var id int64 + if err := pool.QueryRow(ctx, `INSERT INTO imap_accounts (owner, name, host, username, password_enc, tenant_id) + VALUES ($1, 'n', 'imap.example.test', 'u', '\x00', $2) RETURNING id`, owner, tenant).Scan(&id); err != nil { + t.Fatal(err) + } + return id + } + acc1 = mk("alice", 1) + acc2 = mk("bob", 1) + accNoTenant = mk("carol", nil) + + if _, err := pool.Exec(ctx, ` + INSERT INTO emails (id, message_id, tenant_id, source_type, source_id) VALUES + ('m-legacy', 'legacy@example.test', 1, NULL, NULL), + ('m-ref', 'ref@example.test', NULL, 'smtp', NULL), + ('m-t2', 't2@example.test', 2, 'imap', 99); + INSERT INTO email_refs (email_id, tenant_id) VALUES ('m-ref', 1); + `); err != nil { + t.Fatal(err) + } + + lookup := NewLinkLookup(pool) + cands, err := lookup(ctx, acc1, []string{"legacy@example.test", "ref@example.test", "t2@example.test", "none@example.test"}) + if err != nil { + t.Fatal(err) + } + if !cands["legacy@example.test"].TenantOK || !cands["ref@example.test"].TenantOK || cands["t2@example.test"].TenantOK { + t.Errorf("tenant classification wrong: %+v", cands) + } + if _, ok := cands["none@example.test"]; ok { + t.Error("unknown message-id must not be a candidate") + } + + write := NewLinkWriter(pool) + mustLink := func(acc int64, id string, want bool) { + t.Helper() + got, err := write(ctx, acc, id) + if err != nil || got != want { + t.Errorf("link(%d, %s): got (%v, %v), want %v", acc, id, got, err, want) + } + } + mustLink(acc1, "m-legacy", true) + mustLink(acc1, "m-legacy", false) + mustLink(acc2, "m-legacy", true) + mustLink(acc1, "m-ref", true) + mustLink(acc1, "m-t2", false) + mustLink(accNoTenant, "m-legacy", false) + + var st, sid any + if err := pool.QueryRow(ctx, `SELECT source_type, source_id FROM emails WHERE id='m-legacy'`).Scan(&st, &sid); err != nil || st != nil || sid != nil { + t.Errorf("source must stay untouched, got (%v, %v, %v)", st, sid, err) + } + + cands, _ = lookup(ctx, acc1, []string{"legacy@example.test"}) + if !cands["legacy@example.test"].Linked { + t.Error("lookup must report existing link") + } + + if _, err := pool.Exec(ctx, `DELETE FROM imap_accounts WHERE id = $1`, acc1); err != nil { + t.Fatal(err) + } + var n int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM imap_account_mails WHERE account_id = $1`, acc1).Scan(&n); err != nil || n != 0 { + t.Errorf("links of a deleted account must cascade, got %d (%v)", n, err) + } + if err := pool.QueryRow(ctx, `SELECT count(*) FROM imap_account_mails`).Scan(&n); err != nil || n != 1 { + t.Errorf("other accounts' links must stay, got %d (%v)", n, err) + } +} + +func TestLinkDryRunDoesNotWrite(t *testing.T) { + pool := testSchemaPool(t) + ctx := context.Background() + if err := migrate(ctx, pool); err != nil { + t.Fatal(err) + } + var acc int64 + if err := pool.QueryRow(ctx, `INSERT INTO imap_accounts (owner, name, host, username, password_enc, tenant_id) + VALUES ('alice', 'n', 'imap.example.test', 'u', '\x00', 1) RETURNING id`).Scan(&acc); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO emails (id, message_id, tenant_id) VALUES ('m1', 'a@example.test', 1)`); err != nil { + t.Fatal(err) + } + src := &fakeSource{ + folders: []VerifyFolder{{Name: "INBOX"}}, + messages: map[string][]VerifyMessage{"INBOX": {{UID: 1, Header: hdr("")}}}, + } + rep := LinkAccount(ctx, acc, src, NewLinkLookup(pool), nil, nil, "", nil) + if rep.New != 1 { + t.Fatalf("dry run must report one candidate, got %+v", rep.LinkCounts) + } + var n int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM imap_account_mails`).Scan(&n); err != nil || n != 0 { + t.Errorf("dry run wrote %d links (%v)", n, err) + } +} diff --git a/internal/imap/importer.go b/internal/imap/importer.go index c3de018..8849060 100644 --- a/internal/imap/importer.go +++ b/internal/imap/importer.go @@ -271,6 +271,12 @@ func (imp *Importer) storeAndIndex(raw []byte, tenantID *int64, accountID int64, log.Warn("failed to tag source", "id", id, "err", err) } + // PROJ-94: also for Message-ID/SHA-256 duplicates; must run before + // DeliveryRcpt so the index already carries this account's token. + if _, err := imp.store.LinkMail(ctx, accountID, id); err != nil { + log.Warn("failed to link mail to imap account", "id", id, "err", err) + } + // Parse for indexing (reuse the early parse from routing-rule resolution). if parseErr != nil { log.Warn("failed to parse mail for indexing", "id", id, "err", parseErr) diff --git a/internal/imap/link.go b/internal/imap/link.go new file mode 100644 index 0000000..a693d70 --- /dev/null +++ b/internal/imap/link.go @@ -0,0 +1,189 @@ +package imap + +import ( + "context" + "fmt" +) + +const ( + LinkOK = "ok" + LinkFailed = "failed" +) + +type LinkCandidate struct { + MailID string + TenantOK bool + Linked bool +} + +type LinkLookup func(ctx context.Context, accountID int64, messageIDs []string) (map[string]LinkCandidate, error) + +type LinkCounts struct { + InMailbox int `json:"in_mailbox"` + New int `json:"new"` + AlreadyLinked int `json:"already_linked"` + NotInArchive int `json:"not_in_archive"` + NoMessageID int `json:"no_message_id"` + SkippedTenant int `json:"skipped_tenant"` +} + +func (c *LinkCounts) add(o LinkCounts) { + c.InMailbox += o.InMailbox + c.New += o.New + c.AlreadyLinked += o.AlreadyLinked + c.NotInArchive += o.NotInArchive + c.NoMessageID += o.NoMessageID + c.SkippedTenant += o.SkippedTenant +} + +type LinkFolderReport struct { + Folder string `json:"folder"` + Status string `json:"status"` + LinkCounts + Error string `json:"error,omitempty"` +} + +type LinkAccountReport struct { + AccountID int64 `json:"account_id"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + LinkCounts + FoldersFailed int `json:"folders_failed"` + Folders []LinkFolderReport `json:"folders"` +} + +type linkRun struct { + accountID int64 + src VerifySource + lookup LinkLookup + write LinkWriter + secrets []string + seen map[string]bool +} + +// write == nil is the dry run: nothing is written, but counts are identical +// to what --apply would produce. +func LinkAccount(ctx context.Context, accountID int64, src VerifySource, lookup LinkLookup, write LinkWriter, excludedFolders []string, onlyFolder string, secrets []string) LinkAccountReport { + rep := LinkAccountReport{AccountID: accountID} + + folders, err := src.Folders() + if err != nil { + rep.Status = LinkFailed + rep.Error = RedactSecrets(fmt.Sprintf("list folders: %v", err), secrets) + return rep + } + + excluded := make(map[string]bool, len(excludedFolders)) + for _, f := range excludedFolders { + excluded[f] = true + } + + run := &linkRun{accountID: accountID, src: src, lookup: lookup, write: write, secrets: secrets, seen: map[string]bool{}} + matched := false + for _, f := range folders { + if onlyFolder != "" && f.Name != onlyFolder { + continue + } + matched = true + switch { + case excluded[f.Name]: + rep.Folders = append(rep.Folders, LinkFolderReport{Folder: f.Name, Status: VerifyExcluded}) + case f.NoSelect: + rep.Folders = append(rep.Folders, LinkFolderReport{Folder: f.Name, Status: VerifyNoSelect}) + default: + rep.Folders = append(rep.Folders, run.folder(ctx, f.Name)) + } + } + if onlyFolder != "" && !matched { + rep.Folders = append(rep.Folders, LinkFolderReport{Folder: onlyFolder, Status: LinkFailed, Error: "folder not found on server"}) + } + + for _, f := range rep.Folders { + rep.add(f.LinkCounts) + if f.Status == LinkFailed { + rep.FoldersFailed++ + } + } + rep.Status = LinkOK + if rep.FoldersFailed > 0 { + rep.Status = LinkFailed + } + return rep +} + +func (r *linkRun) folder(ctx context.Context, name string) LinkFolderReport { + fr := LinkFolderReport{Folder: name, Status: LinkOK} + msgs, err := r.src.Messages(name) + if err != nil { + return LinkFolderReport{Folder: name, Status: LinkFailed, Error: RedactSecrets(err.Error(), r.secrets)} + } + fr.InMailbox = len(msgs) + + ids := make([]string, 0, len(msgs)) + msgIDs := make([]string, len(msgs)) + uniq := make(map[string]bool, len(msgs)) + for i, m := range msgs { + id := NormalizeMessageID(m.Header) + msgIDs[i] = id + if id != "" && !uniq[id] { + uniq[id] = true + ids = append(ids, id) + } + } + + cands := map[string]LinkCandidate{} + if len(ids) > 0 { + cands, err = r.lookup(ctx, r.accountID, ids) + if err != nil { + return LinkFolderReport{Folder: name, Status: LinkFailed, LinkCounts: LinkCounts{InMailbox: len(msgs)}, + Error: RedactSecrets(fmt.Sprintf("archive lookup: %v", err), r.secrets)} + } + } + + for _, id := range msgIDs { + if id == "" { + fr.NoMessageID++ + continue + } + c, ok := cands[id] + switch { + case !ok || c.MailID == "": + fr.NotInArchive++ + case !c.TenantOK: + fr.SkippedTenant++ + case c.Linked || r.seen[c.MailID]: + fr.AlreadyLinked++ + default: + r.seen[c.MailID] = true + if r.write == nil { + fr.New++ + continue + } + inserted, werr := r.write(ctx, r.accountID, c.MailID) + if werr != nil { + fr.Status = LinkFailed + fr.Error = RedactSecrets(fmt.Sprintf("write link: %v", werr), r.secrets) + return fr + } + if inserted { + fr.New++ + } else { + fr.AlreadyLinked++ + } + } + } + return fr +} + +func LinkExitCode(reports []LinkAccountReport) int { + for _, r := range reports { + if r.Status == LinkFailed { + return 2 + } + } + return 0 +} + +func FailedLinkReport(accountID int64, err error, secrets []string) LinkAccountReport { + return LinkAccountReport{AccountID: accountID, Status: LinkFailed, Error: RedactSecrets(err.Error(), secrets)} +} diff --git a/internal/imap/link_output.go b/internal/imap/link_output.go new file mode 100644 index 0000000..117fece --- /dev/null +++ b/internal/imap/link_output.go @@ -0,0 +1,56 @@ +package imap + +import ( + "encoding/json" + "fmt" + "io" +) + +func WriteLinkJSON(w io.Writer, reports []LinkAccountReport, applied bool) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + if reports == nil { + reports = []LinkAccountReport{} + } + return enc.Encode(map[string]any{"applied": applied, "accounts": reports}) +} + +func WriteLinkText(w io.Writer, reports []LinkAccountReport, applied bool) { + newLabel := "neu zuzuordnen" + if applied { + newLabel = "neu zugeordnet" + } else { + fmt.Fprintln(w, "Trockenlauf: keine Änderungen (mit --apply schreiben)") + } + var tot LinkCounts + failed := 0 + for _, r := range reports { + status := "ok" + if r.Status == LinkFailed { + status = "fehlgeschlagen" + } + fmt.Fprintf(w, "Konto %d: %s\n", r.AccountID, status) + if r.Error != "" { + fmt.Fprintf(w, " Fehler: %s\n", r.Error) + failed++ + } + for _, f := range r.Folders { + switch f.Status { + case VerifyExcluded, VerifyNoSelect: + fmt.Fprintf(w, " %-30s %s\n", f.Folder, verifyStatusLabel[f.Status]) + case LinkFailed: + fmt.Fprintf(w, " %-30s fehlgeschlagen: %s\n", f.Folder, f.Error) + default: + fmt.Fprintf(w, " %-30s %s\n", f.Folder, formatLinkCounts(f.LinkCounts, newLabel)) + } + } + tot.add(r.LinkCounts) + failed += r.FoldersFailed + } + fmt.Fprintf(w, "Gesamt: Konten %d %s fehlgeschlagen %d\n", len(reports), formatLinkCounts(tot, newLabel), failed) +} + +func formatLinkCounts(c LinkCounts, newLabel string) string { + return fmt.Sprintf("Postfach %d %s %d bereits zugeordnet %d nicht im Archiv %d ohne Message-ID %d übersprungen (Tenant) %d", + c.InMailbox, newLabel, c.New, c.AlreadyLinked, c.NotInArchive, c.NoMessageID, c.SkippedTenant) +} diff --git a/internal/imap/link_source.go b/internal/imap/link_source.go new file mode 100644 index 0000000..a1caa25 --- /dev/null +++ b/internal/imap/link_source.go @@ -0,0 +1,48 @@ +package imap + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// Same tenant rule as linkAccountMailSQL, evaluated read-only so the dry run +// reports exactly what --apply would write. +const linkLookupSQL = ` +SELECT e.message_id, e.id, + (e.tenant_id IS NOT DISTINCT FROM a.tenant_id + OR (a.tenant_id IS NOT NULL AND EXISTS ( + SELECT 1 FROM email_refs r WHERE r.email_id = e.id AND r.tenant_id = a.tenant_id))) AS tenant_ok, + EXISTS (SELECT 1 FROM imap_account_mails m WHERE m.account_id = a.id AND m.mail_id = e.id) AS linked +FROM emails e +CROSS JOIN imap_accounts a +WHERE a.id = $1 AND e.message_id = ANY($2) +` + +func NewLinkLookup(pool *pgxpool.Pool) LinkLookup { + return func(ctx context.Context, accountID int64, ids []string) (map[string]LinkCandidate, error) { + out := make(map[string]LinkCandidate, len(ids)) + for i := 0; i < len(ids); i += verifyLookupBatch { + end := min(i+verifyLookupBatch, len(ids)) + rows, err := pool.Query(ctx, linkLookupSQL, accountID, ids[i:end]) + if err != nil { + return nil, fmt.Errorf("query emails: %w", err) + } + for rows.Next() { + var msgID string + var c LinkCandidate + if err := rows.Scan(&msgID, &c.MailID, &c.TenantOK, &c.Linked); err != nil { + rows.Close() + return nil, fmt.Errorf("scan link candidate: %w", err) + } + out[msgID] = c + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("rows: %w", err) + } + } + return out, nil + } +} diff --git a/internal/imap/link_test.go b/internal/imap/link_test.go new file mode 100644 index 0000000..340c75a --- /dev/null +++ b/internal/imap/link_test.go @@ -0,0 +1,146 @@ +package imap + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" +) + +type fakeArchive map[string]LinkCandidate + +func (a fakeArchive) lookup(_ context.Context, _ int64, ids []string) (map[string]LinkCandidate, error) { + out := map[string]LinkCandidate{} + for _, id := range ids { + if c, ok := a[id]; ok { + out[id] = c + } + } + return out, nil +} + +type recordingWriter struct { + calls []string + err error +} + +func (w *recordingWriter) write(_ context.Context, _ int64, mailID string) (bool, error) { + w.calls = append(w.calls, mailID) + return w.err == nil, w.err +} + +func linkFixture() (*fakeSource, fakeArchive) { + src := &fakeSource{ + folders: []VerifyFolder{{Name: "INBOX"}, {Name: "Archiv"}, {Name: "Spam"}, {Name: "[Root]", NoSelect: true}}, + messages: map[string][]VerifyMessage{ + "INBOX": { + {UID: 1, Header: hdr("")}, + {UID: 2, Header: hdr("")}, + {UID: 3, Header: hdr("")}, + {UID: 4, Header: hdr("")}, + {UID: 5, Header: hdr("")}, + }, + "Archiv": { + {UID: 9, Header: hdr("")}, + {UID: 10, Header: hdr("")}, + }, + "Spam": {{UID: 1, Header: hdr("")}}, + }, + } + arch := fakeArchive{ + "new1@example.test": {MailID: "m-new1", TenantOK: true}, + "new2@example.test": {MailID: "m-new2", TenantOK: true}, + "linked@example.test": {MailID: "m-linked", TenantOK: true, Linked: true}, + "othertenant@example.test": {MailID: "m-foreign", TenantOK: false}, + "spam@example.test": {MailID: "m-spam", TenantOK: true}, + } + return src, arch +} + +func TestLinkAccountDryRunCounts(t *testing.T) { + src, arch := linkFixture() + rep := LinkAccount(context.Background(), 7, src, arch.lookup, nil, []string{"Spam"}, "", nil) + + want := LinkCounts{InMailbox: 7, New: 2, AlreadyLinked: 2, NotInArchive: 1, NoMessageID: 1, SkippedTenant: 1} + if rep.LinkCounts != want { + t.Fatalf("counts: got %+v, want %+v", rep.LinkCounts, want) + } + if rep.Status != LinkOK || LinkExitCode([]LinkAccountReport{rep}) != 0 { + t.Errorf("status %q", rep.Status) + } + for _, f := range src.requested { + if f == "Spam" || f == "[Root]" { + t.Errorf("folder %q must not be fetched", f) + } + } +} + +func TestLinkAccountApplyMatchesDryRun(t *testing.T) { + src, arch := linkFixture() + dry := LinkAccount(context.Background(), 7, src, arch.lookup, nil, []string{"Spam"}, "", nil) + + src2, _ := linkFixture() + w := &recordingWriter{} + applied := LinkAccount(context.Background(), 7, src2, arch.lookup, w.write, []string{"Spam"}, "", nil) + + if applied.LinkCounts != dry.LinkCounts { + t.Errorf("apply %+v differs from dry run %+v", applied.LinkCounts, dry.LinkCounts) + } + if strings.Join(w.calls, ",") != "m-new1,m-new2" { + t.Errorf("writes: got %v", w.calls) + } +} + +func TestLinkAccountWriteError(t *testing.T) { + src, arch := linkFixture() + w := &recordingWriter{err: errors.New("permission denied for pw-secret")} + rep := LinkAccount(context.Background(), 7, src, arch.lookup, w.write, nil, "INBOX", []string{"pw-secret"}) + if rep.Status != LinkFailed || LinkExitCode([]LinkAccountReport{rep}) != 2 { + t.Fatalf("write error must fail the account, got %q", rep.Status) + } + if strings.Contains(rep.Folders[0].Error, "pw-secret") { + t.Error("error must be redacted") + } +} + +func TestLinkAccountFailures(t *testing.T) { + src := &fakeSource{folderErr: errors.New("boom")} + if rep := LinkAccount(context.Background(), 1, src, fakeArchive{}.lookup, nil, nil, "", nil); rep.Status != LinkFailed { + t.Error("LIST failure must fail") + } + src2, _ := linkFixture() + failing := func(context.Context, int64, []string) (map[string]LinkCandidate, error) { + return nil, errors.New("db down") + } + if rep := LinkAccount(context.Background(), 1, src2, failing, nil, nil, "INBOX", nil); rep.Status != LinkFailed || rep.New != 0 { + t.Errorf("lookup failure must fail without counting new links: %+v", rep) + } + src3, arch := linkFixture() + if rep := LinkAccount(context.Background(), 1, src3, arch.lookup, nil, nil, "Nope", nil); rep.Status != LinkFailed { + t.Error("unknown folder must fail") + } +} + +func TestLinkOutputHasNoSecrets(t *testing.T) { + src, arch := linkFixture() + src.messageErr = map[string]error{"Archiv": errors.New("login imap-user@example.test with hunter2 rejected")} + secrets := []string{"hunter2", "imap-user@example.test"} + rep := LinkAccount(context.Background(), 7, src, arch.lookup, nil, nil, "", secrets) + + var text, js bytes.Buffer + WriteLinkText(&text, []LinkAccountReport{rep}, false) + if err := WriteLinkJSON(&js, []LinkAccountReport{rep}, false); err != nil { + t.Fatal(err) + } + for _, out := range []string{text.String(), js.String()} { + for _, bad := range []string{"hunter2", "imap-user@example.test", "@example.test", "m-new1", "new1"} { + if strings.Contains(out, bad) { + t.Errorf("output leaks %q:\n%s", bad, out) + } + } + } + if !strings.Contains(text.String(), "Trockenlauf") { + t.Error("dry run must be labelled") + } +} diff --git a/internal/imap/store.go b/internal/imap/store.go index 9958450..4d05652 100644 --- a/internal/imap/store.go +++ b/internal/imap/store.go @@ -107,25 +107,30 @@ func New(dsn, secret string) (*Store, error) { return nil, fmt.Errorf("imap store: connect: %w", err) } - if _, err := pool.Exec(context.Background(), createTableSQL); err != nil { + if err := migrate(context.Background(), pool); err != nil { pool.Close() - return nil, fmt.Errorf("imap store: migrate create: %w", err) - } - - if _, err := pool.Exec(context.Background(), migrationSQL); err != nil { - pool.Close() - return nil, fmt.Errorf("imap store: migrate alter: %w", err) - } - - if _, err := pool.Exec(context.Background(), folderStateSQL); err != nil { - pool.Close() - return nil, fmt.Errorf("imap store: migrate folder state: %w", err) + return nil, err } key := sha256.Sum256([]byte(secret)) return &Store{pool: pool, encKey: key}, nil } +func migrate(ctx context.Context, pool *pgxpool.Pool) error { + steps := []struct{ name, sql string }{ + {"create", createTableSQL}, + {"alter", migrationSQL}, + {"folder state", folderStateSQL}, + {"account mails", accountMailsSQL}, + } + for _, st := range steps { + if _, err := pool.Exec(ctx, st.sql); err != nil { + return fmt.Errorf("imap store: migrate %s: %w", st.name, err) + } + } + return nil +} + // Close releases the database connection pool. func (s *Store) Close() { s.pool.Close() diff --git a/internal/storage/delivery_rcpt.go b/internal/storage/delivery_rcpt.go index 9e49598..4086e9e 100644 --- a/internal/storage/delivery_rcpt.go +++ b/internal/storage/delivery_rcpt.go @@ -15,15 +15,18 @@ import ( // may belong to a Message-ID duplicate whose headers differ from the stored // copy; in that case the stored copy is parsed instead. PROJ-90: IMAP mails // additionally carry the account token, even without delivery addresses. +// PROJ-94: plus one token per linked account; a failed link lookup only drops +// those tokens, the detail check re-reads the links anyway. func (s *Store) DeliveryRcpt(ctx context.Context, id string, raw []byte, pm *mailparser.ParsedMail) string { src, err := s.GetMailSource(ctx, id) if err != nil { return "" } - parts := s.trustedDeliveryAddrs(id, raw, pm, src.Type) - if tok := src.AccountToken(); tok != "" { - parts = append(parts, tok) + if linked, lerr := s.LinkedImapAccounts(ctx, id); lerr == nil { + src.Linked = linked } + parts := s.trustedDeliveryAddrs(id, raw, pm, src.Type) + parts = append(parts, src.AccountTokens()...) return strings.Join(parts, " ") } diff --git a/internal/storage/mail_links.go b/internal/storage/mail_links.go new file mode 100644 index 0000000..15432e8 --- /dev/null +++ b/internal/storage/mail_links.go @@ -0,0 +1,39 @@ +package storage + +import ( + "context" + "fmt" +) + +func (s *Store) LinkedImapAccounts(ctx context.Context, id string) ([]int64, error) { + m, err := s.LinkedImapAccountsBatch(ctx, []string{id}) + if err != nil { + return nil, err + } + return m[id], nil +} + +func (s *Store) LinkedImapAccountsBatch(ctx context.Context, ids []string) (map[string][]int64, error) { + out := make(map[string][]int64, len(ids)) + if s.db == nil || len(ids) == 0 { + return out, nil + } + rows, err := s.db.Query(ctx, + `SELECT mail_id, account_id FROM imap_account_mails WHERE mail_id = ANY($1) ORDER BY mail_id, account_id`, ids) + if err != nil { + return nil, fmt.Errorf("storage: linked imap accounts: %w", err) + } + defer rows.Close() + for rows.Next() { + var mid string + var aid int64 + if err := rows.Scan(&mid, &aid); err != nil { + return nil, fmt.Errorf("storage: linked imap accounts scan: %w", err) + } + out[mid] = append(out[mid], aid) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("storage: linked imap accounts rows: %w", err) + } + return out, nil +} diff --git a/internal/storage/mail_source.go b/internal/storage/mail_source.go index 0bfdaf3..4cdfe57 100644 --- a/internal/storage/mail_source.go +++ b/internal/storage/mail_source.go @@ -16,6 +16,9 @@ const imapAccountTokenPrefix = "imapacct" type MailSource struct { Type string ID *int64 + // PROJ-94: accounts from imap_account_mails, loaded separately because + // a failed link lookup must not drop the source-based rules. + Linked []int64 } func ImapAccountToken(accountID int64) string { @@ -40,6 +43,29 @@ func (m MailSource) AccountToken() string { return ImapAccountToken(id) } +func (m MailSource) AccountIDs() []int64 { + var ids []int64 + if id, ok := m.ImapAccountID(); ok { + ids = append(ids, id) + } + for _, id := range m.Linked { + if id > 0 { + ids = append(ids, id) + } + } + slices.Sort(ids) + return slices.Compact(ids) +} + +func (m MailSource) AccountTokens() []string { + ids := m.AccountIDs() + out := make([]string, 0, len(ids)) + for _, id := range ids { + out = append(out, ImapAccountToken(id)) + } + return out +} + // Unknown IDs and legacy mails yield an empty MailSource without error. func (s *Store) GetMailSource(ctx context.Context, id string) (MailSource, error) { if s.db == nil { diff --git a/internal/storage/mail_source_test.go b/internal/storage/mail_source_test.go index 96742a8..cb62ddf 100644 --- a/internal/storage/mail_source_test.go +++ b/internal/storage/mail_source_test.go @@ -68,3 +68,34 @@ func TestImapAccountIDsForOwnerWithoutDB(t *testing.T) { t.Errorf("no DB must yield no rcpt, got %q", rcpt) } } + +func TestMailSourceAccountTokensWithLinks(t *testing.T) { + id := func(v int64) *int64 { return &v } + tests := []struct { + name string + src MailSource + want []string + }{ + {"source only", MailSource{Type: "imap", ID: id(3)}, []string{"imapacct3"}}, + {"links without source", MailSource{Linked: []int64{12, 4}}, []string{"imapacct4", "imapacct12"}}, + {"source plus links, deduplicated", MailSource{Type: "imap", ID: id(4), Linked: []int64{12, 4, 4}}, []string{"imapacct4", "imapacct12"}}, + {"pop3 source is no account", MailSource{Type: "pop3", ID: id(4), Linked: []int64{7}}, []string{"imapacct7"}}, + {"invalid link ids", MailSource{Linked: []int64{0, -1}}, []string{}}, + {"nothing", MailSource{}, []string{}}, + } + for _, tt := range tests { + if got := tt.src.AccountTokens(); !slices.Equal(got, tt.want) { + t.Errorf("%s: got %v, want %v", tt.name, got, tt.want) + } + } +} + +func TestLinkedImapAccountsWithoutDB(t *testing.T) { + s := &Store{} + if ids, err := s.LinkedImapAccounts(context.Background(), "x"); err != nil || ids != nil { + t.Errorf("got (%v, %v)", ids, err) + } + if m, err := s.LinkedImapAccountsBatch(context.Background(), []string{"x"}); err != nil || len(m) != 0 { + t.Errorf("got (%v, %v)", m, err) + } +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 83e0121..cab0e17 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -758,6 +758,13 @@ func (s *Store) Delete(id string) error { tenantIDs, _ = s.TenantsForMail(ctx, id) + // imap_account_mails is created by the imap store; a CLI run against + // a database that never started the daemon must not fail the delete. + var hasLinks bool + if err := s.db.QueryRow(ctx, `SELECT to_regclass('imap_account_mails') IS NOT NULL`).Scan(&hasLinks); err != nil { + return fmt.Errorf("storage: delete: check imap_account_mails: %w", err) + } + tx, err := s.db.Begin(ctx) if err != nil { return fmt.Errorf("storage: delete: begin tx: %w", err) @@ -770,6 +777,11 @@ func (s *Store) Delete(id string) error { if _, err := tx.Exec(ctx, `DELETE FROM email_attachments WHERE email_id = $1`, id); err != nil { return fmt.Errorf("storage: delete: email_attachments: %w", err) } + if hasLinks { + if _, err := tx.Exec(ctx, `DELETE FROM imap_account_mails WHERE mail_id = $1`, id); err != nil { + return fmt.Errorf("storage: delete: imap_account_mails: %w", err) + } + } if _, err := tx.Exec(ctx, `DELETE FROM emails WHERE id = $1`, id); err != nil { return fmt.Errorf("storage: delete: emails: %w", err) } diff --git a/src/data/features.ts b/src/data/features.ts index df5383d..518374b 100644 --- a/src/data/features.ts +++ b/src/data/features.ts @@ -97,4 +97,5 @@ export const features: Feature[] = [ { id: "PROJ-91", name: "Envelope-Absender (Return-Path) durchsuchbar machen", status: "In Review", frontend: false, backend: true, lastUpdated: "2026-10-29", version: "1.0" }, { id: "PROJ-92", name: "Weiterleitung in Liste und Detail anzeigen", status: "In Review", frontend: true, backend: true, lastUpdated: "2026-10-29", version: "1.0" }, { id: "PROJ-93", name: "IMAP-Vollständigkeitsprüfung (Postfach vs. Archiv)", status: "In Review", frontend: false, backend: true, lastUpdated: "2026-10-29", version: "1.0" }, + { id: "PROJ-94", name: "Postfach-Zuordnung pro IMAP-Konto (Archiv spiegelt Posteingang)", status: "In Progress", frontend: false, backend: true, lastUpdated: "2026-10-29", version: "1.0" }, ];