package mailparser import ( "bytes" "unicode/utf8" "golang.org/x/text/encoding/charmap" ) // Charset repair for mails that carry 8-bit bytes without (or with a wrong) // charset declaration. // // Two real-world cases in the archive that PROJ-84 did NOT cover: // // 1. Subject/From/filename headers with raw 8-bit bytes and no RFC 2047 // encoded-word at all (RFC 5322 forbids it, but many older newsletter and // shop systems do it anyway). Go's mime.WordDecoder passes such a header // through byte-for-byte, so a Windows-1252 "ü" (0xFC) ended up as an // invalid UTF-8 byte in emails.subject and in Manticore. // // 2. text/* body parts without a charset parameter (or declared us-ascii) // that nevertheless contain Latin-1/Windows-1252 bytes. decodeCharset() // treats an empty/us-ascii charset as "already UTF-8" and returns the // bytes unchanged — same broken result. // // RepairUTF8Bytes closes both gaps with the fallback strategy common in mail // clients: assume UTF-8 first, and only for bytes that cannot be part of a // valid UTF-8 sequence fall back to Windows-1252 (a superset of ISO-8859-1, // which is what such senders almost always mean). // // Deliberate properties: // - Input that is already valid UTF-8 is returned untouched (identity), so // correctly encoded mails can never be mangled. // - The repair is byte-wise, not buffer-wise: a header that mixes a properly // decoded UTF-8 encoded-word with raw Latin-1 bytes keeps the UTF-8 part // intact instead of being re-decoded as a whole. // - It only ever touches the derived metadata/search text. The archived // original (encrypted EML) is never rewritten — GoBD immutability. func RepairUTF8Bytes(b []byte) []byte { if utf8.Valid(b) { return b } var out bytes.Buffer out.Grow(len(b) + len(b)/4) for i := 0; i < len(b); { if b[i] < utf8.RuneSelf { out.WriteByte(b[i]) i++ continue } // Prefer a genuine UTF-8 sequence when one is present. if r, size := utf8.DecodeRune(b[i:]); r != utf8.RuneError || size > 1 { out.Write(b[i : i+size]) i += size continue } // Not valid UTF-8 at this position: interpret the single byte as // Windows-1252. Undefined CP1252 bytes yield U+FFFD, which is still // better than an invalid byte in the database/index. out.WriteRune(charmap.Windows1252.DecodeByte(b[i])) i++ } return out.Bytes() } // RepairUTF8 is the string form of RepairUTF8Bytes. func RepairUTF8(s string) string { if utf8.ValidString(s) { return s } return string(RepairUTF8Bytes([]byte(s))) } // NeedsCharsetRepair reports whether s contains bytes that are not valid UTF-8 // and would therefore be repaired by RepairUTF8. Metadata repair runs use it to // pick candidate rows; it is intentionally cheap and side-effect free. func NeedsCharsetRepair(s string) bool { return !utf8.ValidString(s) }