feat(PROJ-85): Fix undeklarierte 8-Bit-Zeichen in Header/Body ohne Encoded-Word

Getrennt von PROJ-84: Header (v.a. Subject) mit rohen 8-Bit-Bytes ohne RFC-2047-Encoded-Word-Syntax wurden nicht auf tatsächliches Charset geprüft, landeten als ungültiges UTF-8 in emails.subject und Manticore. Gleiche Lücke bei decodeCharset() für den Body ohne verwertbaren Content-Type. RepairUTF8/RepairUTF8Bytes (charset_repair.go): bytegenaue Reparatur, gültiges UTF-8 bleibt Identität, nur ungültige Byte-Sequenzen fallen auf Windows-1252 zurück. Attachment.Data bewusst ausgenommen (bleibt byte-exakt für Downloads). fix-subjects-Kommando erkennt jetzt beide Fälle (HasEncodedWord || NeedsCharsetRepair). Verifiziert auf 192.168.1.132: 54 zusätzliche Subject-Fälle, 219 Body-Fälle behoben (bodyInvalidUTF8 219 -> 0). --apply noch nicht ausgeführt, Body-Korrektur braucht zusätzlich reindex. Die ursprünglich gemeldete Amazon-Mail bleibt bewusst unverändert: Encoding-Fehler kam bereits so vom Absender (=3F statt =DC im Original-Encoded-Word), GoBD verbietet nachträgliche Korrektur archivierter Originalinhalte. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WapWkrQusDuBMhaN8WyuXB
2026-08-06 20:46:58 +02:00
parent 786d8341f4
commit 8f46d688a8
10 changed files with 369 additions and 12 deletions
@@ -0,0 +1,79 @@
+package mailparser
+
+import (
+	"bytes"
+	"unicode/utf8"
+
+	"golang.org/x/text/encoding/charmap"
+)
+
+// Charset repair for mails that carry 8-bit bytes without (or with a wrong)
+// charset declaration.
+//
+// Two real-world cases in the archive that PROJ-84 did NOT cover:
+//
+//  1. Subject/From/filename headers with raw 8-bit bytes and no RFC 2047
+//     encoded-word at all (RFC 5322 forbids it, but many older newsletter and
+//     shop systems do it anyway). Go's mime.WordDecoder passes such a header
+//     through byte-for-byte, so a Windows-1252 "ü" (0xFC) ended up as an
+//     invalid UTF-8 byte in emails.subject and in Manticore.
+//
+//  2. text/* body parts without a charset parameter (or declared us-ascii)
+//     that nevertheless contain Latin-1/Windows-1252 bytes. decodeCharset()
+//     treats an empty/us-ascii charset as "already UTF-8" and returns the
+//     bytes unchanged — same broken result.
+//
+// RepairUTF8Bytes closes both gaps with the fallback strategy common in mail
+// clients: assume UTF-8 first, and only for bytes that cannot be part of a
+// valid UTF-8 sequence fall back to Windows-1252 (a superset of ISO-8859-1,
+// which is what such senders almost always mean).
+//
+// Deliberate properties:
+//   - Input that is already valid UTF-8 is returned untouched (identity), so
+//     correctly encoded mails can never be mangled.
+//   - The repair is byte-wise, not buffer-wise: a header that mixes a properly
+//     decoded UTF-8 encoded-word with raw Latin-1 bytes keeps the UTF-8 part
+//     intact instead of being re-decoded as a whole.
+//   - It only ever touches the derived metadata/search text. The archived
+//     original (encrypted EML) is never rewritten — GoBD immutability.
+func RepairUTF8Bytes(b []byte) []byte {
+	if utf8.Valid(b) {
+		return b
+	}
+	var out bytes.Buffer
+	out.Grow(len(b) + len(b)/4)
+	for i := 0; i < len(b); {
+		if b[i] < utf8.RuneSelf {
+			out.WriteByte(b[i])
+			i++
+			continue
+		}
+		// Prefer a genuine UTF-8 sequence when one is present.
+		if r, size := utf8.DecodeRune(b[i:]); r != utf8.RuneError || size > 1 {
+			out.Write(b[i : i+size])
+			i += size
+			continue
+		}
+		// Not valid UTF-8 at this position: interpret the single byte as
+		// Windows-1252. Undefined CP1252 bytes yield U+FFFD, which is still
+		// better than an invalid byte in the database/index.
+		out.WriteRune(charmap.Windows1252.DecodeByte(b[i]))
+		i++
+	}
+	return out.Bytes()
+}
+
+// RepairUTF8 is the string form of RepairUTF8Bytes.
+func RepairUTF8(s string) string {
+	if utf8.ValidString(s) {
+		return s
+	}
+	return string(RepairUTF8Bytes([]byte(s)))
+}
+
+// NeedsCharsetRepair reports whether s contains bytes that are not valid UTF-8
+// and would therefore be repaired by RepairUTF8. Metadata repair runs use it to
+// pick candidate rows; it is intentionally cheap and side-effect free.
+func NeedsCharsetRepair(s string) bool {
+	return !utf8.ValidString(s)
+}
@@ -0,0 +1,78 @@
+package mailparser
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestRepairUTF8(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{"pure ascii untouched", "Passwort geaendert", "Passwort geaendert"},
+		{"valid utf8 untouched", "Passwort geändert – ok", "Passwort geändert – ok"},
+		{"windows1252 umlauts", "Passwort ge\xe4ndert", "Passwort geändert"},
+		{"windows1252 sharp s", "Mini-Fu\xdfball", "Mini-Fußball"},
+		{"cp1252 en dash 0x96", "Netbook f\xfcr 207 Euro \x96 jetzt", "Netbook für 207 Euro – jetzt"},
+		{"cp1252 registered 0xae", "NVIDIA\xae Karten", "NVIDIA® Karten"},
+		{"mixed valid utf8 and latin1", "Gr\xfc\xdfe – ünd", "Grüße – ünd"},
+		{"empty", "", ""},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := RepairUTF8(tc.in); got != tc.want {
+				t.Errorf("RepairUTF8(%q) = %q, want %q", tc.in, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestNeedsCharsetRepair(t *testing.T) {
+	if NeedsCharsetRepair("Passwort geändert") {
+		t.Error("valid UTF-8 must not be flagged for repair")
+	}
+	if !NeedsCharsetRepair("Passwort ge\xe4ndert") {
+		t.Error("raw Latin-1 byte must be flagged for repair")
+	}
+}
+
+// A Subject header with raw 8-bit bytes and no RFC 2047 encoded-word must be
+// decoded via the Windows-1252 fallback instead of ending up as invalid UTF-8.
+func TestParseRawEightBitSubject(t *testing.T) {
+	raw := "From: a@example.com\r\n" +
+		"Subject: Passwort f\xfcr WoltLab ge\xe4ndert\r\n" +
+		"Content-Type: text/plain\r\n\r\n" +
+		"Gr\xfc\xdfe\r\n"
+
+	pm, err := Parse([]byte(raw))
+	if err != nil {
+		t.Fatalf("Parse: %v", err)
+	}
+	if pm.Subject != "Passwort für WoltLab geändert" {
+		t.Errorf("Subject = %q", pm.Subject)
+	}
+	if !strings.Contains(pm.TextBody, "Grüße") {
+		t.Errorf("TextBody = %q", pm.TextBody)
+	}
+}
+
+// A correctly encoded UTF-8 mail must survive the repair unchanged.
+func TestParseValidUTF8NotMangled(t *testing.T) {
+	raw := "From: a@example.com\r\n" +
+		"Subject: =?UTF-8?Q?Gr=C3=BC=C3=9Fe?=\r\n" +
+		"Content-Type: text/plain; charset=UTF-8\r\n\r\n" +
+		"Schöne Grüße – äöüß\r\n"
+
+	pm, err := Parse([]byte(raw))
+	if err != nil {
+		t.Fatalf("Parse: %v", err)
+	}
+	if pm.Subject != "Grüße" {
+		t.Errorf("Subject = %q", pm.Subject)
+	}
+	if !strings.Contains(pm.TextBody, "Schöne Grüße – äöüß") {
+		t.Errorf("TextBody = %q", pm.TextBody)
+	}
+}
@@ -197,7 +197,9 @@ func Parse(raw []byte) (pmOut *ParsedMail, errOut error) {
 	if err != nil {
 		// No content-type or parse error: treat as plain text
 		body, _ := io.ReadAll(msg.Body)
-		pm.TextBody = string(body)
+		// Mails without any Content-Type are typically old 8-bit Latin-1
+		// messages — repair them like a declared text/plain part.
+		pm.TextBody = string(RepairUTF8Bytes(body))
 		return pm, nil
 	}

@@ -211,11 +213,16 @@ func Parse(raw []byte) (pmOut *ParsedMail, errOut error) {
 	} else {
 		body, _ := io.ReadAll(msg.Body)
 		decoded := decodeBody(body, msg.Header.Get("Content-Transfer-Encoding"))
-		decoded = decodeCharset(decoded, params["charset"])
+		if strings.HasPrefix(mediaType, "text/") || mediaType == "" {
+			decoded = decodeCharset(decoded, params["charset"])
+		}
+		// The body always ends up as displayed/indexed text here, so the 8-bit
+		// fallback repair applies to every branch (undeclared charset, wrongly
+		// declared charset, or no usable Content-Type). No-op for valid UTF-8.
 		if strings.Contains(mediaType, "html") {
-			pm.HTMLBody = string(decoded)
+			pm.HTMLBody = string(RepairUTF8Bytes(decoded))
 		} else {
-			pm.TextBody = string(decoded)
+			pm.TextBody = string(RepairUTF8Bytes(decoded))
 		}
 	}

@@ -276,11 +283,13 @@ func parseMultipart(pm *ParsedMail, body io.Reader, boundary string, depth int)
 			continue
 		}

+		// Only the displayed/indexed body text gets the 8-bit fallback repair;
+		// attachment bytes above stay byte-exact on purpose.
 		switch {
 		case strings.Contains(mediaType, "text/plain"):
-			pm.textBuf.Write(decoded)
+			pm.textBuf.Write(RepairUTF8Bytes(decoded))
 		case strings.Contains(mediaType, "text/html"):
-			pm.htmlBuf.Write(decoded)
+			pm.htmlBuf.Write(RepairUTF8Bytes(decoded))
 		}
 	}
 }
@@ -340,7 +349,13 @@ func decodeMIMEHeader(s string) string {
 	}
 	decoded, err := dec.DecodeHeader(s)
 	if err != nil {
-		return s
+		// Undecodable encoded-word: keep the raw value, but still repair raw
+		// 8-bit bytes so nothing invalid reaches the DB/index.
+		return RepairUTF8(s)
 	}
-	return decoded
+	// Headers without any encoded-word are returned byte-for-byte by
+	// WordDecoder. Senders that write raw Windows-1252/ISO-8859-1 bytes into
+	// the header would otherwise land as invalid UTF-8 in emails.subject.
+	// RepairUTF8 is a no-op for valid UTF-8.
+	return RepairUTF8(decoded)
 }