feat(PROJ-85): Fix undeklarierte 8-Bit-Zeichen in Header/Body ohne Encoded-Word
Getrennt von PROJ-84: Header (v.a. Subject) mit rohen 8-Bit-Bytes ohne RFC-2047-Encoded-Word-Syntax wurden nicht auf tatsächliches Charset geprüft, landeten als ungültiges UTF-8 in emails.subject und Manticore. Gleiche Lücke bei decodeCharset() für den Body ohne verwertbaren Content-Type. RepairUTF8/RepairUTF8Bytes (charset_repair.go): bytegenaue Reparatur, gültiges UTF-8 bleibt Identität, nur ungültige Byte-Sequenzen fallen auf Windows-1252 zurück. Attachment.Data bewusst ausgenommen (bleibt byte-exakt für Downloads). fix-subjects-Kommando erkennt jetzt beide Fälle (HasEncodedWord || NeedsCharsetRepair). Verifiziert auf 192.168.1.132: 54 zusätzliche Subject-Fälle, 219 Body-Fälle behoben (bodyInvalidUTF8 219 -> 0). --apply noch nicht ausgeführt, Body-Korrektur braucht zusätzlich reindex. Die ursprünglich gemeldete Amazon-Mail bleibt bewusst unverändert: Encoding-Fehler kam bereits so vom Absender (=3F statt =DC im Original-Encoded-Word), GoBD verbietet nachträgliche Korrektur archivierter Originalinhalte. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WapWkrQusDuBMhaN8WyuXB
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
786d8341f4
commit
8f46d688a8
@@ -197,7 +197,9 @@ func Parse(raw []byte) (pmOut *ParsedMail, errOut error) {
|
||||
if err != nil {
|
||||
// No content-type or parse error: treat as plain text
|
||||
body, _ := io.ReadAll(msg.Body)
|
||||
pm.TextBody = string(body)
|
||||
// Mails without any Content-Type are typically old 8-bit Latin-1
|
||||
// messages — repair them like a declared text/plain part.
|
||||
pm.TextBody = string(RepairUTF8Bytes(body))
|
||||
return pm, nil
|
||||
}
|
||||
|
||||
@@ -211,11 +213,16 @@ func Parse(raw []byte) (pmOut *ParsedMail, errOut error) {
|
||||
} else {
|
||||
body, _ := io.ReadAll(msg.Body)
|
||||
decoded := decodeBody(body, msg.Header.Get("Content-Transfer-Encoding"))
|
||||
decoded = decodeCharset(decoded, params["charset"])
|
||||
if strings.HasPrefix(mediaType, "text/") || mediaType == "" {
|
||||
decoded = decodeCharset(decoded, params["charset"])
|
||||
}
|
||||
// The body always ends up as displayed/indexed text here, so the 8-bit
|
||||
// fallback repair applies to every branch (undeclared charset, wrongly
|
||||
// declared charset, or no usable Content-Type). No-op for valid UTF-8.
|
||||
if strings.Contains(mediaType, "html") {
|
||||
pm.HTMLBody = string(decoded)
|
||||
pm.HTMLBody = string(RepairUTF8Bytes(decoded))
|
||||
} else {
|
||||
pm.TextBody = string(decoded)
|
||||
pm.TextBody = string(RepairUTF8Bytes(decoded))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -276,11 +283,13 @@ func parseMultipart(pm *ParsedMail, body io.Reader, boundary string, depth int)
|
||||
continue
|
||||
}
|
||||
|
||||
// Only the displayed/indexed body text gets the 8-bit fallback repair;
|
||||
// attachment bytes above stay byte-exact on purpose.
|
||||
switch {
|
||||
case strings.Contains(mediaType, "text/plain"):
|
||||
pm.textBuf.Write(decoded)
|
||||
pm.textBuf.Write(RepairUTF8Bytes(decoded))
|
||||
case strings.Contains(mediaType, "text/html"):
|
||||
pm.htmlBuf.Write(decoded)
|
||||
pm.htmlBuf.Write(RepairUTF8Bytes(decoded))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -340,7 +349,13 @@ func decodeMIMEHeader(s string) string {
|
||||
}
|
||||
decoded, err := dec.DecodeHeader(s)
|
||||
if err != nil {
|
||||
return s
|
||||
// Undecodable encoded-word: keep the raw value, but still repair raw
|
||||
// 8-bit bytes so nothing invalid reaches the DB/index.
|
||||
return RepairUTF8(s)
|
||||
}
|
||||
return decoded
|
||||
// Headers without any encoded-word are returned byte-for-byte by
|
||||
// WordDecoder. Senders that write raw Windows-1252/ISO-8859-1 bytes into
|
||||
// the header would otherwise land as invalid UTF-8 in emails.subject.
|
||||
// RepairUTF8 is a no-op for valid UTF-8.
|
||||
return RepairUTF8(decoded)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user