fix(PROJ-57): UTF-8-Encoding für Mails mit Nicht-UTF-8-Charset korrigieren

Der Mail-Parser ignorierte das charset-Parameter aus Content-Type und
interpretierte Bytes immer als UTF-8, wodurch iso-8859-1/windows-1252
kodierte Mails (z.B. mit Umlauten) als Mojibake gespeichert wurden.
Zusätzlich fehlte das Charset für die Manticore-MySQL-Verbindung und
der charset-Parameter im JSON-Response-Header.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

cmd/archivmail/cmd_import.go

@@ -83,7 +83,7 @@ func runImport(args []string) {
 	if backend == "manticore" {
 		dsn := cfg.Index.ManticoreDSN
 		if dsn == "" {
-			dsn = "manticore@tcp(127.0.0.1:9306)/"
+			dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4"
 		}
 		m, err := index.NewManticoreTenantManager(dsn)
 		if err != nil {

cmd/archivmail/cmd_import_piler.go

@@ -87,7 +87,7 @@ func runImportPiler(args []string) {
 	if backend == "manticore" {
 		dsn := cfg.Index.ManticoreDSN
 		if dsn == "" {
-			dsn = "manticore@tcp(127.0.0.1:9306)/"
+			dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4"
 		}
 		m, err := index.NewManticoreTenantManager(dsn)
 		if err != nil {

cmd/archivmail/cmd_ocr_reprocess.go

@@ -61,7 +61,7 @@ func runOCRReprocess(args []string) {
 	}
 	dsn := cfg.Index.ManticoreDSN
 	if dsn == "" {
-		dsn = "manticore@tcp(127.0.0.1:9306)/"
+		dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4"
 	}
 	idxMgr, err := index.NewManticoreTenantManager(dsn)
 	if err != nil {

cmd/archivmail/cmd_purge.go

@@ -78,7 +78,7 @@ func runPurge(args []string) {
 	if indexBackend == "manticore" {
 		dsn := cfg.Index.ManticoreDSN
 		if dsn == "" {
-			dsn = "manticore@tcp(127.0.0.1:9306)/"
+			dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4"
 		}
 		if m, err := index.NewManticoreTenantManager(dsn); err == nil {
 			idxMgr = m

cmd/archivmail/cmd_reindex.go

@@ -55,7 +55,7 @@ func runReindex(args []string) {
 	if indexBackend == "manticore" {
 		dsn := cfg.Index.ManticoreDSN
 		if dsn == "" {
-			dsn = "manticore@tcp(127.0.0.1:9306)/"
+			dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4"
 		}
 		m, err := index.NewManticoreTenantManager(dsn)
 		if err != nil {

cmd/archivmail/cmd_status.go

@@ -116,7 +116,7 @@ func checkPostgres(cfg *config.Config) checkResult {
 func checkManticore(cfg *config.Config) checkResult {
 	dsn := cfg.Index.ManticoreDSN
 	if dsn == "" {
-		dsn = "manticore@tcp(127.0.0.1:9306)/"
+		dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4"
 	}
 
 	start := time.Now()

cmd/archivmail/main.go

@@ -156,7 +156,7 @@ func main() {
 	if indexBackend == "manticore" {
 		dsn := cfg.Index.ManticoreDSN
 		if dsn == "" {
-			dsn = "manticore@tcp(127.0.0.1:9306)/"
+			dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4"
 		}
 		m, err := index.NewManticoreTenantManager(dsn)
 		if err != nil {

config/config.go

@@ -156,7 +156,7 @@ type IndexConfig struct {
 	Backend        string `yaml:"backend"`
 	BatchSize      int    `yaml:"batch_size"`
 	AsyncQueueSize int    `yaml:"async_queue_size"`
-	ManticoreDSN   string `yaml:"manticore_dsn"` // DSN for Manticore backend (default: "manticore@tcp(127.0.0.1:9306)/")
+	ManticoreDSN   string `yaml:"manticore_dsn"` // DSN for Manticore backend (default: "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4")
 }
 
 // DefaultAuditLogPath is the default location of the append-only JSON-Lines

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/jackc/pgx/v5 v5.6.0
 	github.com/pquerna/otp v1.4.0
 	golang.org/x/crypto v0.48.0
+	golang.org/x/text v0.34.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -30,5 +31,4 @@ require (
 	github.com/kr/text v0.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
 )

internal/api/server.go

@@ -412,7 +412,7 @@ func (s *Server) requireMailAccess(next http.HandlerFunc) http.HandlerFunc {
 // --- helpers ---
 
 func writeJSON(w http.ResponseWriter, code int, v interface{}) {
-	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	w.WriteHeader(code)
 	json.NewEncoder(w).Encode(v)
 }

pkg/mailparser/parser.go

@@ -11,6 +11,8 @@ import (
 	"net/mail"
 	"strings"
 	"time"
+
+	"golang.org/x/text/encoding/htmlindex"
 )
 
 // Attachment represents a MIME attachment in a parsed email.
@@ -184,6 +186,7 @@ func Parse(raw []byte) (*ParsedMail, error) {
 	} else {
 		body, _ := io.ReadAll(msg.Body)
 		decoded := decodeBody(body, msg.Header.Get("Content-Transfer-Encoding"))
+		decoded = decodeCharset(decoded, params["charset"])
 		if strings.Contains(mediaType, "html") {
 			pm.HTMLBody = string(decoded)
 		} else {
@@ -216,6 +219,9 @@ func parseMultipart(pm *ParsedMail, body io.Reader, boundary string) error {
 		data, _ := io.ReadAll(part)
 		cte := part.Header.Get("Content-Transfer-Encoding")
 		decoded := decodeBody(data, cte)
+		if strings.Contains(mediaType, "text/") {
+			decoded = decodeCharset(decoded, params["charset"])
+		}
 
 		// Check disposition for attachment
 		disp := part.Header.Get("Content-Disposition")
@@ -254,6 +260,24 @@ func parseMultipart(pm *ParsedMail, body io.Reader, boundary string) error {
 	return nil
 }
 
+// decodeCharset converts data from the declared MIME charset to UTF-8.
+// Empty charset or "utf-8"/"us-ascii" are passed through unchanged.
+func decodeCharset(data []byte, charset string) []byte {
+	charset = strings.ToLower(strings.TrimSpace(charset))
+	if charset == "" || charset == "utf-8" || charset == "us-ascii" || charset == "ascii" {
+		return data
+	}
+	enc, err := htmlindex.Get(charset)
+	if err != nil {
+		return data
+	}
+	decoded, err := enc.NewDecoder().Bytes(data)
+	if err != nil {
+		return data
+	}
+	return decoded
+}
+
 // decodeBody decodes Content-Transfer-Encoding if needed.
 func decodeBody(data []byte, cte string) []byte {
 	switch strings.ToLower(strings.TrimSpace(cte)) {