feat(PROJ-85): Fix undeklarierte 8-Bit-Zeichen in Header/Body ohne Encoded-Word

Getrennt von PROJ-84: Header (v.a. Subject) mit rohen 8-Bit-Bytes ohne
RFC-2047-Encoded-Word-Syntax wurden nicht auf tatsächliches Charset
geprüft, landeten als ungültiges UTF-8 in emails.subject und Manticore.
Gleiche Lücke bei decodeCharset() für den Body ohne verwertbaren
Content-Type.

RepairUTF8/RepairUTF8Bytes (charset_repair.go): bytegenaue Reparatur,
gültiges UTF-8 bleibt Identität, nur ungültige Byte-Sequenzen fallen
auf Windows-1252 zurück. Attachment.Data bewusst ausgenommen (bleibt
byte-exakt für Downloads). fix-subjects-Kommando erkennt jetzt beide
Fälle (HasEncodedWord || NeedsCharsetRepair).

Verifiziert auf 192.168.1.132: 54 zusätzliche Subject-Fälle, 219
Body-Fälle behoben (bodyInvalidUTF8 219 -> 0). --apply noch nicht
ausgeführt, Body-Korrektur braucht zusätzlich reindex.

Die ursprünglich gemeldete Amazon-Mail bleibt bewusst unverändert:
Encoding-Fehler kam bereits so vom Absender (=3F statt =DC im
Original-Encoded-Word), GoBD verbietet nachträgliche Korrektur
archivierter Originalinhalte.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WapWkrQusDuBMhaN8WyuXB
This commit is contained in:
sysops
2026-08-06 20:46:58 +02:00
co-authored by Claude Sonnet 5
parent 786d8341f4
commit 8f46d688a8
10 changed files with 369 additions and 12 deletions
+23 -8
View File
@@ -197,7 +197,9 @@ func Parse(raw []byte) (pmOut *ParsedMail, errOut error) {
if err != nil {
// No content-type or parse error: treat as plain text
body, _ := io.ReadAll(msg.Body)
pm.TextBody = string(body)
// Mails without any Content-Type are typically old 8-bit Latin-1
// messages — repair them like a declared text/plain part.
pm.TextBody = string(RepairUTF8Bytes(body))
return pm, nil
}
@@ -211,11 +213,16 @@ func Parse(raw []byte) (pmOut *ParsedMail, errOut error) {
} else {
body, _ := io.ReadAll(msg.Body)
decoded := decodeBody(body, msg.Header.Get("Content-Transfer-Encoding"))
decoded = decodeCharset(decoded, params["charset"])
if strings.HasPrefix(mediaType, "text/") || mediaType == "" {
decoded = decodeCharset(decoded, params["charset"])
}
// The body always ends up as displayed/indexed text here, so the 8-bit
// fallback repair applies to every branch (undeclared charset, wrongly
// declared charset, or no usable Content-Type). No-op for valid UTF-8.
if strings.Contains(mediaType, "html") {
pm.HTMLBody = string(decoded)
pm.HTMLBody = string(RepairUTF8Bytes(decoded))
} else {
pm.TextBody = string(decoded)
pm.TextBody = string(RepairUTF8Bytes(decoded))
}
}
@@ -276,11 +283,13 @@ func parseMultipart(pm *ParsedMail, body io.Reader, boundary string, depth int)
continue
}
// Only the displayed/indexed body text gets the 8-bit fallback repair;
// attachment bytes above stay byte-exact on purpose.
switch {
case strings.Contains(mediaType, "text/plain"):
pm.textBuf.Write(decoded)
pm.textBuf.Write(RepairUTF8Bytes(decoded))
case strings.Contains(mediaType, "text/html"):
pm.htmlBuf.Write(decoded)
pm.htmlBuf.Write(RepairUTF8Bytes(decoded))
}
}
}
@@ -340,7 +349,13 @@ func decodeMIMEHeader(s string) string {
}
decoded, err := dec.DecodeHeader(s)
if err != nil {
return s
// Undecodable encoded-word: keep the raw value, but still repair raw
// 8-bit bytes so nothing invalid reaches the DB/index.
return RepairUTF8(s)
}
return decoded
// Headers without any encoded-word are returned byte-for-byte by
// WordDecoder. Senders that write raw Windows-1252/ISO-8859-1 bytes into
// the header would otherwise land as invalid UTF-8 in emails.subject.
// RepairUTF8 is a no-op for valid UTF-8.
return RepairUTF8(decoded)
}