feat(PROJ-84): fix-subjects CLI-Backfill für kaputte MIME-Header-Betreffs
CLI-Subkommando "archivmail fix-subjects" korrigiert Bestandsmails mit
undekodiertem RFC-2047-Encoded-Word im Betreff (Folge des Fixes in
31d7113). Dry-Run per Default, echte Änderung nur mit --apply.
Original-EML im Storage bleibt unangetastet - nur Postgres emails.subject
wird korrigiert, danach Manticore-Reindex inkl. Erhalt von OCR-Text.
Jeder Lauf erzeugt einen Audit-Log-Eintrag (metadata_backfill).
Verifiziert auf 192.168.1.132: 543 Kandidaten, 541 reparierbar, 2 zu
Recht übersprungen (RFC-2047-Verletzung im Original). --apply noch
nicht ausgeführt.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WapWkrQusDuBMhaN8WyuXB
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
31d71138c9
commit
1dfd8c18ac
@@ -0,0 +1,319 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"archivmail/config"
|
||||||
|
"archivmail/internal/audit"
|
||||||
|
"archivmail/internal/index"
|
||||||
|
"archivmail/internal/storage"
|
||||||
|
"archivmail/pkg/mailparser"
|
||||||
|
)
|
||||||
|
|
||||||
|
// runFixSubjects repairs `emails.subject` rows that still contain a raw,
|
||||||
|
// undecoded RFC 2047 encoded-word (e.g. `=?Windows-1252?Q?...?=`). Those were
|
||||||
|
// written before decodeMIMEHeader() got a CharsetReader for charsets outside
|
||||||
|
// UTF-8/US-ASCII/ISO-8859-1.
|
||||||
|
//
|
||||||
|
// Scope of the repair:
|
||||||
|
// - ONLY the PostgreSQL metadata column `subject` is rewritten. The archived
|
||||||
|
// original (encrypted EML in the store) is never touched — this is a
|
||||||
|
// display/search metadata repair, not a change to the immutable archive
|
||||||
|
// copy (same principle as PROJ-34 / PROJ-48).
|
||||||
|
// - After a successful DB update the mail is re-indexed in Manticore
|
||||||
|
// (emails_global / emails_tenant_N), otherwise search would keep showing
|
||||||
|
// the old subject.
|
||||||
|
// - The run itself is written to the audit log (system action).
|
||||||
|
//
|
||||||
|
// Dry-run is the DEFAULT: without --apply nothing is written.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
//
|
||||||
|
// archivmail fix-subjects # dry-run over all tenants
|
||||||
|
// archivmail fix-subjects --tenant 2 --limit 50
|
||||||
|
// archivmail fix-subjects --apply
|
||||||
|
func runFixSubjects(args []string) {
|
||||||
|
fs := flag.NewFlagSet("fix-subjects", flag.ExitOnError)
|
||||||
|
configPath := fs.String("config", "/etc/archivmail/config.yml", "path to config file")
|
||||||
|
tenantIDFlag := fs.Int64("tenant", 0, "tenant ID (0 = all tenants)")
|
||||||
|
limitFlag := fs.Int("limit", 0, "max number of mails to inspect (0 = no limit)")
|
||||||
|
applyFlag := fs.Bool("apply", false, "actually write changes (default: dry-run only)")
|
||||||
|
verboseFlag := fs.Bool("verbose", false, "log every affected subject (old -> new)")
|
||||||
|
fs.Parse(args)
|
||||||
|
|
||||||
|
logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||||
|
|
||||||
|
cfg, err := config.Load(*configPath)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("failed to load config", "err", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
storeCfg := storage.Config{
|
||||||
|
Dir: cfg.Storage.StorePath,
|
||||||
|
Keyfile: cfg.Storage.Keyfile,
|
||||||
|
DSN: cfg.Database.DSN(),
|
||||||
|
CompressEnabled: cfg.Storage.Compress,
|
||||||
|
}
|
||||||
|
mailStore, err := storage.New(storeCfg)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("storage init failed", "err", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
defer mailStore.Close()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
var tenantPtr *int64
|
||||||
|
if *tenantIDFlag > 0 {
|
||||||
|
t := *tenantIDFlag
|
||||||
|
tenantPtr = &t
|
||||||
|
}
|
||||||
|
|
||||||
|
// Index manager is only needed when we actually write.
|
||||||
|
var idxMgr index.TenantIndexer
|
||||||
|
if *applyFlag {
|
||||||
|
idxMgr, err = openIndexManager(cfg)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("index init failed", "err", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
defer func() { idxMgr.Close() }()
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := mailStore.ListRawEncodedSubjects(ctx, tenantPtr, *limitFlag)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("failed to list candidate subjects", "err", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
mode := "DRY-RUN"
|
||||||
|
if *applyFlag {
|
||||||
|
mode = "APPLY"
|
||||||
|
}
|
||||||
|
logger.Info("fix-subjects: starting",
|
||||||
|
"mode", mode, "tenant", *tenantIDFlag, "candidates", len(rows))
|
||||||
|
|
||||||
|
var (
|
||||||
|
affected int // subject really contains an encoded-word and decodes differently
|
||||||
|
updated int
|
||||||
|
undecodable int // still encoded after decoding attempt
|
||||||
|
reindexed int
|
||||||
|
indexSkipped int
|
||||||
|
errCount int
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, row := range rows {
|
||||||
|
if !mailparser.HasEncodedWord(row.Subject) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Preferred source of truth: re-parse the archived original (read-only).
|
||||||
|
// Falls back to decoding the stored string if the mail cannot be read.
|
||||||
|
newSubject := ""
|
||||||
|
var pm *mailparser.ParsedMail
|
||||||
|
var raw []byte
|
||||||
|
if data, lerr := mailStore.Load(row.ID); lerr == nil {
|
||||||
|
if parsed, perr := mailparser.Parse(data); perr == nil {
|
||||||
|
pm = parsed
|
||||||
|
raw = data
|
||||||
|
newSubject = parsed.Subject
|
||||||
|
} else {
|
||||||
|
logger.Warn("fix-subjects: parse failed, falling back to header decode",
|
||||||
|
"id", row.ID, "err", perr)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
logger.Warn("fix-subjects: load failed, falling back to header decode",
|
||||||
|
"id", row.ID, "err", lerr)
|
||||||
|
}
|
||||||
|
if newSubject == "" {
|
||||||
|
newSubject = mailparser.DecodeMIMEHeader(row.Subject)
|
||||||
|
}
|
||||||
|
|
||||||
|
if newSubject == row.Subject {
|
||||||
|
undecodable++
|
||||||
|
logger.Warn("fix-subjects: subject still not decodable", "id", row.ID, "subject", row.Subject)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
affected++
|
||||||
|
|
||||||
|
if *verboseFlag || !*applyFlag {
|
||||||
|
logger.Info("fix-subjects: would fix",
|
||||||
|
"id", row.ID, "tenant", tenantLabel(row.TenantID),
|
||||||
|
"old", row.Subject, "new", newSubject)
|
||||||
|
}
|
||||||
|
if !*applyFlag {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := mailStore.UpdateSubjectMetadata(ctx, row.ID, newSubject); err != nil {
|
||||||
|
logger.Warn("fix-subjects: update failed", "id", row.ID, "err", err)
|
||||||
|
errCount++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
updated++
|
||||||
|
|
||||||
|
// Pull the corrected subject into the full-text index.
|
||||||
|
if pm == nil {
|
||||||
|
indexSkipped++
|
||||||
|
logger.Warn("fix-subjects: mail not parseable — index not updated", "id", row.ID)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := reindexOne(ctx, mailStore, idxMgr, row.ID, row.TenantID, pm, raw); err != nil {
|
||||||
|
logger.Warn("fix-subjects: reindex failed", "id", row.ID, "err", err)
|
||||||
|
errCount++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
reindexed++
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Info("fix-subjects: complete",
|
||||||
|
"mode", mode,
|
||||||
|
"candidates", len(rows),
|
||||||
|
"affected", affected,
|
||||||
|
"updated", updated,
|
||||||
|
"reindexed", reindexed,
|
||||||
|
"index_skipped", indexSkipped,
|
||||||
|
"undecodable", undecodable,
|
||||||
|
"errors", errCount)
|
||||||
|
|
||||||
|
writeFixSubjectsAudit(cfg, logger, fixSubjectsAudit{
|
||||||
|
Apply: *applyFlag,
|
||||||
|
TenantID: tenantPtr,
|
||||||
|
Candidates: len(rows),
|
||||||
|
Affected: affected,
|
||||||
|
Updated: updated,
|
||||||
|
Reindexed: reindexed,
|
||||||
|
IndexSkipped: indexSkipped,
|
||||||
|
Undecodable: undecodable,
|
||||||
|
Errors: errCount,
|
||||||
|
})
|
||||||
|
|
||||||
|
if errCount > 0 {
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// openIndexManager builds the configured index backend (manticore by default).
|
||||||
|
func openIndexManager(cfg *config.Config) (index.TenantIndexer, error) {
|
||||||
|
backend := cfg.Index.Backend
|
||||||
|
if backend == "" {
|
||||||
|
backend = "manticore"
|
||||||
|
}
|
||||||
|
if backend == "manticore" {
|
||||||
|
dsn := cfg.Index.ManticoreDSN
|
||||||
|
if dsn == "" {
|
||||||
|
dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4"
|
||||||
|
}
|
||||||
|
m, err := index.NewManticoreTenantManager(dsn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("manticore init: %w", err)
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
batchSize := cfg.Index.BatchSize
|
||||||
|
if batchSize <= 0 {
|
||||||
|
batchSize = 100
|
||||||
|
}
|
||||||
|
m, err := index.NewTenantIndexManager(cfg.Index.Path, batchSize, backend)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("index manager init: %w", err)
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// reindexOne re-indexes a single mail from an already parsed message. Existing
|
||||||
|
// OCR text is preserved because IndexSync uses REPLACE (see runReindex).
|
||||||
|
func reindexOne(ctx context.Context, store *storage.Store, mgr index.TenantIndexer,
|
||||||
|
id string, tenantID *int64, pm *mailparser.ParsedMail, raw []byte) error {
|
||||||
|
|
||||||
|
var attachNames []string
|
||||||
|
for _, a := range pm.Attachments {
|
||||||
|
if a.Filename != "" {
|
||||||
|
attachNames = append(attachNames, a.Filename)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
doc := index.MailDocument{
|
||||||
|
ID: id,
|
||||||
|
From: pm.From,
|
||||||
|
To: strings.Join(pm.To, ", "),
|
||||||
|
CC: strings.Join(pm.CC, ", "),
|
||||||
|
Subject: pm.Subject,
|
||||||
|
Body: pm.TextBody,
|
||||||
|
AttachNames: strings.Join(attachNames, " "),
|
||||||
|
HasAttachment: len(pm.Attachments) > 0,
|
||||||
|
Date: pm.Date,
|
||||||
|
Size: int64(len(raw)),
|
||||||
|
TenantID: tenantID,
|
||||||
|
}
|
||||||
|
|
||||||
|
idx := mgr.ForTenant(tenantID)
|
||||||
|
|
||||||
|
if _, ocrChars, err := store.GetOCRMeta(ctx, id); err == nil && ocrChars > 0 {
|
||||||
|
if reader, ok := idx.(index.AttachmentTextReader); ok {
|
||||||
|
if existing, rerr := reader.GetAttachmentText(id); rerr == nil && existing != "" {
|
||||||
|
doc.AttachmentText = existing
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := idx.IndexSync(doc); err != nil {
|
||||||
|
return fmt.Errorf("index sync: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fixSubjectsAudit carries the counters of one backfill run into the audit log.
|
||||||
|
type fixSubjectsAudit struct {
|
||||||
|
Apply bool
|
||||||
|
TenantID *int64
|
||||||
|
Candidates int
|
||||||
|
Affected int
|
||||||
|
Updated int
|
||||||
|
Reindexed int
|
||||||
|
IndexSkipped int
|
||||||
|
Undecodable int
|
||||||
|
Errors int
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeFixSubjectsAudit records the run (dry-run included) in the audit log.
|
||||||
|
// Audit failures must never abort the maintenance run itself.
|
||||||
|
func writeFixSubjectsAudit(cfg *config.Config, logger *slog.Logger, a fixSubjectsAudit) {
|
||||||
|
audlog, err := audit.New(cfg.Database.DSN(), cfg.Audit.ResolvedLogPath(), logger)
|
||||||
|
if err != nil {
|
||||||
|
logger.Warn("fix-subjects: audit log unavailable", "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer audlog.Close()
|
||||||
|
|
||||||
|
mode := "dry-run"
|
||||||
|
if a.Apply {
|
||||||
|
mode = "apply"
|
||||||
|
}
|
||||||
|
audlog.Log(audit.Entry{
|
||||||
|
EventType: audit.EventMetadataBackfill,
|
||||||
|
Username: "system",
|
||||||
|
IPAddress: "cli",
|
||||||
|
Query: "fix-subjects",
|
||||||
|
Success: a.Errors == 0,
|
||||||
|
TenantID: a.TenantID,
|
||||||
|
Detail: fmt.Sprintf(
|
||||||
|
"fix-subjects (%s): candidates=%d affected=%d updated=%d reindexed=%d index_skipped=%d undecodable=%d errors=%d; "+
|
||||||
|
"nur emails.subject korrigiert, Archivoriginal unverändert",
|
||||||
|
mode, a.Candidates, a.Affected, a.Updated, a.Reindexed, a.IndexSkipped, a.Undecodable, a.Errors),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// tenantLabel renders a nullable tenant ID for log output.
|
||||||
|
func tenantLabel(t *int64) string {
|
||||||
|
if t == nil {
|
||||||
|
return "global"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%d", *t)
|
||||||
|
}
|
||||||
@@ -308,6 +308,7 @@ Commands:
|
|||||||
purge Mails mit abgelaufener Aufbewahrungsfrist löschen (cron-fähig)
|
purge Mails mit abgelaufener Aufbewahrungsfrist löschen (cron-fähig)
|
||||||
recompress Bestehende Mails nachträglich gzip-komprimieren
|
recompress Bestehende Mails nachträglich gzip-komprimieren
|
||||||
rethread Thread-IDs rückwirkend aus In-Reply-To/References befüllen
|
rethread Thread-IDs rückwirkend aus In-Reply-To/References befüllen
|
||||||
|
fix-subjects Kaputte RFC-2047-Betreffs in den Metadaten reparieren (Dry-Run per Default)
|
||||||
ocr-reprocess OCR für Anhänge nachholen (alle oder pro Mandant/Status)
|
ocr-reprocess OCR für Anhänge nachholen (alle oder pro Mandant/Status)
|
||||||
index-pending Ungeindexte Mails nachindexieren (cron-fähig, PROJ-58 batch_mode)
|
index-pending Ungeindexte Mails nachindexieren (cron-fähig, PROJ-58 batch_mode)
|
||||||
backup Store, Keyfile, PostgreSQL und Config konsistent sichern (PROJ-66)
|
backup Store, Keyfile, PostgreSQL und Config konsistent sichern (PROJ-66)
|
||||||
|
|||||||
@@ -78,6 +78,9 @@ func main() {
|
|||||||
case "rethread":
|
case "rethread":
|
||||||
runRethread(os.Args[2:])
|
runRethread(os.Args[2:])
|
||||||
return
|
return
|
||||||
|
case "fix-subjects":
|
||||||
|
runFixSubjects(os.Args[2:])
|
||||||
|
return
|
||||||
case "ocr-reprocess":
|
case "ocr-reprocess":
|
||||||
runOCRReprocess(os.Args[2:])
|
runOCRReprocess(os.Args[2:])
|
||||||
return
|
return
|
||||||
|
|||||||
+2
-1
@@ -99,7 +99,8 @@
|
|||||||
| PROJ-81 | Anhang-Online-Vorschau (PDF, Bilder) | In Review | [PROJ-81](PROJ-81-anhang-online-vorschau.md) | 2026-08-06 |
|
| PROJ-81 | Anhang-Online-Vorschau (PDF, Bilder) | In Review | [PROJ-81](PROJ-81-anhang-online-vorschau.md) | 2026-08-06 |
|
||||||
| PROJ-82 | Print-Farbparität zwischen Hell- und Dark-Mode-Ausdrucken | Planned | [PROJ-82](PROJ-82-print-farbparitaet-dark-mode.md) | 2026-08-06 |
|
| PROJ-82 | Print-Farbparität zwischen Hell- und Dark-Mode-Ausdrucken | Planned | [PROJ-82](PROJ-82-print-farbparitaet-dark-mode.md) | 2026-08-06 |
|
||||||
| PROJ-83 | Audit-Logging für Anhang-Abrufe (GoBD/DSGVO-Nachbesserung) | Planned | [PROJ-83](PROJ-83-audit-log-anhang-abrufe.md) | 2026-08-06 |
|
| PROJ-83 | Audit-Logging für Anhang-Abrufe (GoBD/DSGVO-Nachbesserung) | Planned | [PROJ-83](PROJ-83-audit-log-anhang-abrufe.md) | 2026-08-06 |
|
||||||
|
| PROJ-84 | Fix MIME-Header-Charset-Dekodierung + Backfill für Bestandsmails | In Review | [PROJ-84](PROJ-84-fix-mime-header-charset-backfill.md) | 2026-08-06 |
|
||||||
|
|
||||||
<!-- Add features above this line -->
|
<!-- Add features above this line -->
|
||||||
|
|
||||||
## Next Available ID: PROJ-84
|
## Next Available ID: PROJ-85
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
# PROJ-84: Fix MIME-Header-Charset-Dekodierung + Backfill für Bestandsmails
|
||||||
|
|
||||||
|
## Status: In Review
|
||||||
|
**Created:** 2026-08-06
|
||||||
|
**Last Updated:** 2026-08-06
|
||||||
|
|
||||||
|
## Kontext
|
||||||
|
Betreffzeilen mit MIME-Encoded-Words (RFC 2047) in Charsets außerhalb UTF-8/US-ASCII/ISO-8859-1 (z.B. Windows-1252, ISO-8859-15) wurden nicht dekodiert und roh angezeigt (`=?Windows-1252?Q?ARAG_4_...?=`). Ursache: `decodeMIMEHeader()` in `pkg/mailparser/parser.go` nutzte `mime.WordDecoder` ohne `CharsetReader` — Go unterstützt dort nativ nur die drei genannten Charsets. Der Body-Decoder (`decodeCharset`, genutzt seit PROJ-57) konnte das bereits über `htmlindex`, der Header-Decoder nicht.
|
||||||
|
|
||||||
|
Fix (Commit 31d7113) behebt das für künftige Importe. Bestandsmails behalten den kaputten Betreff, bis sie per Backfill korrigiert werden — dafür dieses Ticket.
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
- Baut auf PROJ-57 (UTF-8-Encoding-Fix) auf — nutzt dieselbe `htmlindex`-Charset-Logik
|
||||||
|
- Betrifft PROJ-11/PROJ-48 (Audit-Log) — Backfill-Lauf wird auditiert
|
||||||
|
|
||||||
|
## User Stories
|
||||||
|
- Als User will ich, dass Mail-Betreffs unabhängig vom ursprünglichen Absender-Charset korrekt lesbar sind, sowohl bei neuen als auch bei bereits archivierten Mails.
|
||||||
|
- Als Admin will ich einen nachvollziehbaren, wiederholbaren Weg haben, bestehende Mails mit kaputtem Betreff zu korrigieren, ohne das archivierte Original anzufassen.
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
- [x] `decodeMIMEHeader()` dekodiert Encoded-Words in Windows-1252, ISO-8859-15 und weiteren von `htmlindex` unterstützten Charsets korrekt
|
||||||
|
- [x] CLI-Kommando `archivmail fix-subjects` identifiziert Bestandsmails mit undekodiertem Encoded-Word im Betreff
|
||||||
|
- [x] Dry-Run ist Default, echte Änderung nur mit `--apply`
|
||||||
|
- [x] Korrektur ändert ausschließlich die Postgres-Metadaten-Spalte `emails.subject` — die archivierte Original-EML im Storage-Layer bleibt unverändert
|
||||||
|
- [x] Nach Korrektur wird der Manticore-Suchindex für die betroffene Mail nachgezogen (inkl. Erhalt von vorhandenem OCR-Text)
|
||||||
|
- [x] Jeder Lauf (auch Dry-Run) erzeugt einen Audit-Log-Eintrag (`event_type=metadata_backfill`) mit Zählern
|
||||||
|
- [ ] `--apply`-Lauf auf 132 durchgeführt und stichprobenartig verifiziert
|
||||||
|
- [ ] `--apply`-Lauf auf 131 (Produktiv) durchgeführt, nach Freigabe
|
||||||
|
|
||||||
|
## Edge Cases
|
||||||
|
- Encoded-Word verletzt RFC 2047 selbst (z.B. Leerzeichen im codierten Teil, an Vortext geklebt) → wird übersprungen, mit WARN geloggt, bleibt unverändert (2 von 543 Fällen auf 132)
|
||||||
|
- Mail-Original nicht mehr ladbar/parsebar (z.B. DSGVO-gelöscht) → Index wird nicht angefasst, als `index_skipped` gezählt, DB-Subject bleibt beim Fallback (`DecodeMIMEHeader` auf vorhandenem String)
|
||||||
|
- Mail bereits mit OCR-Text indexiert → Reindex darf `attachment_text` nicht löschen, wird aus bestehendem Index-Dokument übernommen
|
||||||
|
|
||||||
|
## Technical Requirements (optional)
|
||||||
|
- Kein neues Datenbankschema, keine neue Tabelle
|
||||||
|
- CLI-Subkommando statt Wegwerf-Skript, für künftige ähnliche Encoding-Bugs wiederverwendbar
|
||||||
|
- Exit-Code 1 bei Fehlern (cron-/scripttauglich)
|
||||||
|
|
||||||
|
---
|
||||||
|
<!-- Sections below are added by subsequent skills -->
|
||||||
|
|
||||||
|
## Tech Design (Solution Architect)
|
||||||
|
Direkt umgesetzt ohne vorgelagerte Architektur-Phase — kleiner, klar umrissener Bugfix + Backfill-Tool, kein neues UI, kein neues Datenmodell.
|
||||||
|
|
||||||
|
Quelle der Wahrheit ist das archivierte Original: pro Mail wird die verschlüsselte EML lesend geladen und mit `mailparser.Parse()` neu geparst, der neue Betreff kommt aus `pm.Subject`. Fallback auf direktes Redekodieren des gespeicherten Strings, falls Original nicht ladbar. Geschrieben wird ausschließlich `UPDATE emails SET subject=...`.
|
||||||
|
|
||||||
|
## Implementation Notes (Backend, 2026-08-06)
|
||||||
|
|
||||||
|
Neue/geänderte Dateien:
|
||||||
|
- `pkg/mailparser/header_decode.go` (neu) — `DecodeMIMEHeader()` exportierter Wrapper, `HasEncodedWord()` Erkennung
|
||||||
|
- `internal/storage/subject_backfill.go` (neu) — `ListRawEncodedSubjects()`, `UpdateSubjectMetadata()`
|
||||||
|
- `cmd/archivmail/cmd_fix_subjects.go` (neu) — CLI-Kommando inkl. Reindex + Audit
|
||||||
|
- `internal/audit/audit.go` — neue Konstante `EventMetadataBackfill = "metadata_backfill"`
|
||||||
|
- `cmd/archivmail/main.go`, `cmd/archivmail/cmd_import.go` — Dispatch + Hilfetext
|
||||||
|
|
||||||
|
Kommando:
|
||||||
|
```bash
|
||||||
|
archivmail fix-subjects # Dry-Run (Default), alle Mandanten
|
||||||
|
archivmail fix-subjects --tenant 3 --limit 50
|
||||||
|
archivmail fix-subjects --apply
|
||||||
|
archivmail fix-subjects --apply --verbose # + jede Änderung alt->neu loggen
|
||||||
|
```
|
||||||
|
|
||||||
|
Messung auf 192.168.1.132 (Dry-Run, read-only): 543 Kandidaten, 541 reparierbar, 2 nicht dekodierbar (RFC-2047-Verletzung im Original, korrekt übersprungen). Charset-Verteilung: ISO-8859-15 (219), windows-1252 (185), Cp1252 (69), Windows-1252 (38), iso-8859-15 (11), utf8-Varianten (14), windows-1258 (5), ASCII (1).
|
||||||
|
|
||||||
|
Build auf 132 verifiziert (`CGO_ENABLED=0 go build -buildvcs=false ./cmd/archivmail/` → OK), Dry-Run zweimal gegen Live-DB gelaufen, Stichproben korrekt (u.a. mehrteilige Windows-1252-Encoded-Words richtig zusammengesetzt). `--apply` noch nicht ausgeführt.
|
||||||
|
|
||||||
|
## QA Test Results
|
||||||
|
_To be added by /qa_
|
||||||
|
|
||||||
|
## Deployment
|
||||||
|
_To be added by /deploy_
|
||||||
@@ -32,6 +32,11 @@ const (
|
|||||||
// day dropped significantly below its trailing 7-day average, or the IMAP
|
// day dropped significantly below its trailing 7-day average, or the IMAP
|
||||||
// soll/ist comparison revealed a shortfall.
|
// soll/ist comparison revealed a shortfall.
|
||||||
EventReconciliationAnomaly = "reconciliation_anomaly"
|
EventReconciliationAnomaly = "reconciliation_anomaly"
|
||||||
|
// EventMetadataBackfill: a CLI/system maintenance run repaired derived
|
||||||
|
// metadata columns (e.g. `emails.subject` after the RFC 2047 charset fix).
|
||||||
|
// The archived original is never modified by such a run — the entry
|
||||||
|
// documents scope, mode (dry-run/apply) and counters for traceability.
|
||||||
|
EventMetadataBackfill = "metadata_backfill"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Entry is a single audit log record.
|
// Entry is a single audit log record.
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SubjectRow is a minimal projection of an email used by metadata repair runs
|
||||||
|
// (e.g. the fix-subjects backfill). It never carries mail content — the
|
||||||
|
// encrypted original in the store is not touched by such runs.
|
||||||
|
type SubjectRow struct {
|
||||||
|
ID string
|
||||||
|
TenantID *int64
|
||||||
|
Subject string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListRawEncodedSubjects returns emails whose subject still contains an
|
||||||
|
// RFC 2047 encoded-word pattern (`=?charset?B|Q?...?=`). The SQL LIKE is only
|
||||||
|
// a cheap prefilter; the caller must verify with mailparser.HasEncodedWord and
|
||||||
|
// decide whether decoding actually yields a different value.
|
||||||
|
//
|
||||||
|
// tenantID nil = all tenants. limit <= 0 = no limit.
|
||||||
|
func (s *Store) ListRawEncodedSubjects(ctx context.Context, tenantID *int64, limit int) ([]SubjectRow, error) {
|
||||||
|
if s.db == nil {
|
||||||
|
return nil, fmt.Errorf("storage: list raw encoded subjects: no database configured")
|
||||||
|
}
|
||||||
|
|
||||||
|
query := `SELECT id, tenant_id, COALESCE(subject, '')
|
||||||
|
FROM emails
|
||||||
|
WHERE subject LIKE '%=?%?=%'`
|
||||||
|
args := []interface{}{}
|
||||||
|
if tenantID != nil {
|
||||||
|
args = append(args, *tenantID)
|
||||||
|
query += fmt.Sprintf(" AND tenant_id = $%d", len(args))
|
||||||
|
}
|
||||||
|
query += " ORDER BY received_at ASC"
|
||||||
|
if limit > 0 {
|
||||||
|
args = append(args, limit)
|
||||||
|
query += fmt.Sprintf(" LIMIT $%d", len(args))
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := s.db.Query(ctx, query, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("storage: list raw encoded subjects: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var out []SubjectRow
|
||||||
|
for rows.Next() {
|
||||||
|
var r SubjectRow
|
||||||
|
if err := rows.Scan(&r.ID, &r.TenantID, &r.Subject); err != nil {
|
||||||
|
return nil, fmt.Errorf("storage: list raw encoded subjects: scan: %w", err)
|
||||||
|
}
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("storage: list raw encoded subjects: rows: %w", err)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateSubjectMetadata rewrites only the `subject` metadata column of an
|
||||||
|
// email. The archived original (encrypted EML in the store) is deliberately
|
||||||
|
// left untouched — this is a display/search metadata repair, not a change to
|
||||||
|
// the immutable archive copy.
|
||||||
|
func (s *Store) UpdateSubjectMetadata(ctx context.Context, id, subject string) error {
|
||||||
|
if s.db == nil {
|
||||||
|
return fmt.Errorf("storage: update subject metadata: no database configured")
|
||||||
|
}
|
||||||
|
if _, err := s.db.Exec(ctx, `UPDATE emails SET subject = $1 WHERE id = $2`, subject, id); err != nil {
|
||||||
|
return fmt.Errorf("storage: update subject metadata %s: %w", id, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package mailparser
|
||||||
|
|
||||||
|
import "regexp"
|
||||||
|
|
||||||
|
// encodedWordRe matches an RFC 2047 encoded-word (=?charset?B|Q?text?=).
|
||||||
|
// Used to detect header values that were stored raw because decoding failed
|
||||||
|
// (e.g. Windows-1252 before the CharsetReader fix in decodeMIMEHeader).
|
||||||
|
var encodedWordRe = regexp.MustCompile(`=\?[^?\s]+\?[BbQq]\?[^?]*\?=`)
|
||||||
|
|
||||||
|
// HasEncodedWord reports whether s still contains an undecoded RFC 2047
|
||||||
|
// encoded-word. A correctly decoded subject never contains one.
|
||||||
|
func HasEncodedWord(s string) bool {
|
||||||
|
return encodedWordRe.MatchString(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeMIMEHeader is the exported form of decodeMIMEHeader. It decodes
|
||||||
|
// RFC 2047 encoded-word headers including charsets outside UTF-8/US-ASCII/
|
||||||
|
// ISO-8859-1 (e.g. Windows-1252). Callers outside the parser need it for
|
||||||
|
// metadata repair runs (subject backfill).
|
||||||
|
func DecodeMIMEHeader(s string) string {
|
||||||
|
return decodeMIMEHeader(s)
|
||||||
|
}
|
||||||
@@ -87,4 +87,5 @@ export const features: Feature[] = [
|
|||||||
{ id: "PROJ-81", name: "Anhang-Online-Vorschau (PDF, Bilder)", status: "In Review", frontend: true, backend: false, lastUpdated: "2026-08-06", version: "1.0" },
|
{ id: "PROJ-81", name: "Anhang-Online-Vorschau (PDF, Bilder)", status: "In Review", frontend: true, backend: false, lastUpdated: "2026-08-06", version: "1.0" },
|
||||||
{ id: "PROJ-82", name: "Print-Farbparität zwischen Hell- und Dark-Mode-Ausdrucken", status: "Planned", frontend: true, backend: false, lastUpdated: "2026-08-06", version: "1.0" },
|
{ id: "PROJ-82", name: "Print-Farbparität zwischen Hell- und Dark-Mode-Ausdrucken", status: "Planned", frontend: true, backend: false, lastUpdated: "2026-08-06", version: "1.0" },
|
||||||
{ id: "PROJ-83", name: "Audit-Logging für Anhang-Abrufe (GoBD/DSGVO-Nachbesserung)", status: "Planned", frontend: false, backend: true, lastUpdated: "2026-08-06", version: "1.0" },
|
{ id: "PROJ-83", name: "Audit-Logging für Anhang-Abrufe (GoBD/DSGVO-Nachbesserung)", status: "Planned", frontend: false, backend: true, lastUpdated: "2026-08-06", version: "1.0" },
|
||||||
|
{ id: "PROJ-84", name: "Fix MIME-Header-Charset-Dekodierung + Backfill für Bestandsmails", status: "In Review", frontend: false, backend: true, lastUpdated: "2026-08-06", version: "1.0" },
|
||||||
];
|
];
|
||||||
|
|||||||
Reference in New Issue
Block a user