feat(PROJ-85): Fix undeklarierte 8-Bit-Zeichen in Header/Body ohne Encoded-Word
Getrennt von PROJ-84: Header (v.a. Subject) mit rohen 8-Bit-Bytes ohne RFC-2047-Encoded-Word-Syntax wurden nicht auf tatsächliches Charset geprüft, landeten als ungültiges UTF-8 in emails.subject und Manticore. Gleiche Lücke bei decodeCharset() für den Body ohne verwertbaren Content-Type. RepairUTF8/RepairUTF8Bytes (charset_repair.go): bytegenaue Reparatur, gültiges UTF-8 bleibt Identität, nur ungültige Byte-Sequenzen fallen auf Windows-1252 zurück. Attachment.Data bewusst ausgenommen (bleibt byte-exakt für Downloads). fix-subjects-Kommando erkennt jetzt beide Fälle (HasEncodedWord || NeedsCharsetRepair). Verifiziert auf 192.168.1.132: 54 zusätzliche Subject-Fälle, 219 Body-Fälle behoben (bodyInvalidUTF8 219 -> 0). --apply noch nicht ausgeführt, Body-Korrektur braucht zusätzlich reindex. Die ursprünglich gemeldete Amazon-Mail bleibt bewusst unverändert: Encoding-Fehler kam bereits so vom Absender (=3F statt =DC im Original-Encoded-Word), GoBD verbietet nachträgliche Korrektur archivierter Originalinhalte. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WapWkrQusDuBMhaN8WyuXB
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
786d8341f4
commit
8f46d688a8
@@ -86,7 +86,7 @@ func runFixSubjects(args []string) {
|
||||
defer func() { idxMgr.Close() }()
|
||||
}
|
||||
|
||||
rows, err := mailStore.ListRawEncodedSubjects(ctx, tenantPtr, *limitFlag)
|
||||
rows, err := collectSubjectCandidates(ctx, mailStore, tenantPtr, *limitFlag)
|
||||
if err != nil {
|
||||
logger.Error("failed to list candidate subjects", "err", err)
|
||||
os.Exit(1)
|
||||
@@ -109,7 +109,11 @@ func runFixSubjects(args []string) {
|
||||
)
|
||||
|
||||
for _, row := range rows {
|
||||
if !mailparser.HasEncodedWord(row.Subject) {
|
||||
// Two independent defect classes are repaired here:
|
||||
// 1. undecoded RFC 2047 encoded-word (PROJ-84)
|
||||
// 2. raw 8-bit Windows-1252/ISO-8859-1 bytes without encoded-word,
|
||||
// stored as invalid UTF-8
|
||||
if !mailparser.HasEncodedWord(row.Subject) && !mailparser.NeedsCharsetRepair(row.Subject) {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"archivmail/internal/storage"
|
||||
)
|
||||
|
||||
// collectSubjectCandidates merges the two candidate sets the fix-subjects
|
||||
// backfill has to consider and removes duplicates (a subject can be affected by
|
||||
// both defects at once):
|
||||
//
|
||||
// - ListRawEncodedSubjects: subjects that still contain an undecoded RFC 2047
|
||||
// encoded-word (`=?charset?B|Q?...?=`) — the PROJ-84 class.
|
||||
// - ListNonASCIISubjects: subjects with raw 8-bit bytes. A separate query is
|
||||
// unavoidable because such headers carry no `=?...?=` marker at all, so the
|
||||
// encoded-word LIKE filter can never match them.
|
||||
//
|
||||
// limit <= 0 means no limit; otherwise it caps the merged result so that
|
||||
// --limit stays a predictable upper bound on inspected mails.
|
||||
func collectSubjectCandidates(ctx context.Context, store *storage.Store, tenantID *int64, limit int) ([]storage.SubjectRow, error) {
|
||||
encoded, err := store.ListRawEncodedSubjects(ctx, tenantID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nonASCII, err := store.ListNonASCIISubjects(ctx, tenantID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{}, len(encoded)+len(nonASCII))
|
||||
out := make([]storage.SubjectRow, 0, len(encoded)+len(nonASCII))
|
||||
for _, list := range [][]storage.SubjectRow{encoded, nonASCII} {
|
||||
for _, r := range list {
|
||||
if _, dup := seen[r.ID]; dup {
|
||||
continue
|
||||
}
|
||||
seen[r.ID] = struct{}{}
|
||||
out = append(out, r)
|
||||
if limit > 0 && len(out) >= limit {
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
Reference in New Issue
Block a user