From 76655f78a2ec31fb68dd96dabd412cdc712c56f3 Mon Sep 17 00:00:00 2001 From: sysops Date: Wed, 24 Jun 2026 22:47:00 +0200 Subject: [PATCH] =?UTF-8?q?fix(PROJ-57):=20UTF-8-Encoding=20f=C3=BCr=20Mai?= =?UTF-8?q?ls=20mit=20Nicht-UTF-8-Charset=20korrigieren?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Der Mail-Parser ignorierte das charset-Parameter aus Content-Type und interpretierte Bytes immer als UTF-8, wodurch iso-8859-1/windows-1252 kodierte Mails (z.B. mit Umlauten) als Mojibake gespeichert wurden. Zusätzlich fehlte das Charset für die Manticore-MySQL-Verbindung und der charset-Parameter im JSON-Response-Header. Co-Authored-By: Claude Sonnet 4.6 --- cmd/archivmail/cmd_import.go | 2 +- cmd/archivmail/cmd_import_piler.go | 2 +- cmd/archivmail/cmd_ocr_reprocess.go | 2 +- cmd/archivmail/cmd_purge.go | 2 +- cmd/archivmail/cmd_reindex.go | 2 +- cmd/archivmail/cmd_status.go | 2 +- cmd/archivmail/main.go | 2 +- config/config.go | 2 +- go.mod | 2 +- internal/api/server.go | 2 +- pkg/mailparser/parser.go | 24 ++++++++++++++++++++++++ 11 files changed, 34 insertions(+), 10 deletions(-) diff --git a/cmd/archivmail/cmd_import.go b/cmd/archivmail/cmd_import.go index 14e9ecd..ae79697 100644 --- a/cmd/archivmail/cmd_import.go +++ b/cmd/archivmail/cmd_import.go @@ -83,7 +83,7 @@ func runImport(args []string) { if backend == "manticore" { dsn := cfg.Index.ManticoreDSN if dsn == "" { - dsn = "manticore@tcp(127.0.0.1:9306)/" + dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4" } m, err := index.NewManticoreTenantManager(dsn) if err != nil { diff --git a/cmd/archivmail/cmd_import_piler.go b/cmd/archivmail/cmd_import_piler.go index 111dbde..045801c 100644 --- a/cmd/archivmail/cmd_import_piler.go +++ b/cmd/archivmail/cmd_import_piler.go @@ -87,7 +87,7 @@ func runImportPiler(args []string) { if backend == "manticore" { dsn := cfg.Index.ManticoreDSN if dsn == "" { - dsn = "manticore@tcp(127.0.0.1:9306)/" + dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4" } m, err := index.NewManticoreTenantManager(dsn) if err != nil { diff --git a/cmd/archivmail/cmd_ocr_reprocess.go b/cmd/archivmail/cmd_ocr_reprocess.go index f0c102b..ff1b201 100644 --- a/cmd/archivmail/cmd_ocr_reprocess.go +++ b/cmd/archivmail/cmd_ocr_reprocess.go @@ -61,7 +61,7 @@ func runOCRReprocess(args []string) { } dsn := cfg.Index.ManticoreDSN if dsn == "" { - dsn = "manticore@tcp(127.0.0.1:9306)/" + dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4" } idxMgr, err := index.NewManticoreTenantManager(dsn) if err != nil { diff --git a/cmd/archivmail/cmd_purge.go b/cmd/archivmail/cmd_purge.go index 4f1fce2..2388179 100644 --- a/cmd/archivmail/cmd_purge.go +++ b/cmd/archivmail/cmd_purge.go @@ -78,7 +78,7 @@ func runPurge(args []string) { if indexBackend == "manticore" { dsn := cfg.Index.ManticoreDSN if dsn == "" { - dsn = "manticore@tcp(127.0.0.1:9306)/" + dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4" } if m, err := index.NewManticoreTenantManager(dsn); err == nil { idxMgr = m diff --git a/cmd/archivmail/cmd_reindex.go b/cmd/archivmail/cmd_reindex.go index e18cf3c..4c14e45 100644 --- a/cmd/archivmail/cmd_reindex.go +++ b/cmd/archivmail/cmd_reindex.go @@ -55,7 +55,7 @@ func runReindex(args []string) { if indexBackend == "manticore" { dsn := cfg.Index.ManticoreDSN if dsn == "" { - dsn = "manticore@tcp(127.0.0.1:9306)/" + dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4" } m, err := index.NewManticoreTenantManager(dsn) if err != nil { diff --git a/cmd/archivmail/cmd_status.go b/cmd/archivmail/cmd_status.go index 4b535cd..1fc18c5 100644 --- a/cmd/archivmail/cmd_status.go +++ b/cmd/archivmail/cmd_status.go @@ -116,7 +116,7 @@ func checkPostgres(cfg *config.Config) checkResult { func checkManticore(cfg *config.Config) checkResult { dsn := cfg.Index.ManticoreDSN if dsn == "" { - dsn = "manticore@tcp(127.0.0.1:9306)/" + dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4" } start := time.Now() diff --git a/cmd/archivmail/main.go b/cmd/archivmail/main.go index 203c7b6..ce3ce81 100644 --- a/cmd/archivmail/main.go +++ b/cmd/archivmail/main.go @@ -156,7 +156,7 @@ func main() { if indexBackend == "manticore" { dsn := cfg.Index.ManticoreDSN if dsn == "" { - dsn = "manticore@tcp(127.0.0.1:9306)/" + dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4" } m, err := index.NewManticoreTenantManager(dsn) if err != nil { diff --git a/config/config.go b/config/config.go index d040a72..35477a1 100644 --- a/config/config.go +++ b/config/config.go @@ -156,7 +156,7 @@ type IndexConfig struct { Backend string `yaml:"backend"` BatchSize int `yaml:"batch_size"` AsyncQueueSize int `yaml:"async_queue_size"` - ManticoreDSN string `yaml:"manticore_dsn"` // DSN for Manticore backend (default: "manticore@tcp(127.0.0.1:9306)/") + ManticoreDSN string `yaml:"manticore_dsn"` // DSN for Manticore backend (default: "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4") } // DefaultAuditLogPath is the default location of the append-only JSON-Lines diff --git a/go.mod b/go.mod index 58675ac..f0f36ea 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/jackc/pgx/v5 v5.6.0 github.com/pquerna/otp v1.4.0 golang.org/x/crypto v0.48.0 + golang.org/x/text v0.34.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -30,5 +31,4 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect golang.org/x/sync v0.19.0 // indirect - golang.org/x/text v0.34.0 // indirect ) diff --git a/internal/api/server.go b/internal/api/server.go index 345b605..4ac27c5 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -412,7 +412,7 @@ func (s *Server) requireMailAccess(next http.HandlerFunc) http.HandlerFunc { // --- helpers --- func writeJSON(w http.ResponseWriter, code int, v interface{}) { - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(code) json.NewEncoder(w).Encode(v) } diff --git a/pkg/mailparser/parser.go b/pkg/mailparser/parser.go index 3238d76..c554535 100644 --- a/pkg/mailparser/parser.go +++ b/pkg/mailparser/parser.go @@ -11,6 +11,8 @@ import ( "net/mail" "strings" "time" + + "golang.org/x/text/encoding/htmlindex" ) // Attachment represents a MIME attachment in a parsed email. @@ -184,6 +186,7 @@ func Parse(raw []byte) (*ParsedMail, error) { } else { body, _ := io.ReadAll(msg.Body) decoded := decodeBody(body, msg.Header.Get("Content-Transfer-Encoding")) + decoded = decodeCharset(decoded, params["charset"]) if strings.Contains(mediaType, "html") { pm.HTMLBody = string(decoded) } else { @@ -216,6 +219,9 @@ func parseMultipart(pm *ParsedMail, body io.Reader, boundary string) error { data, _ := io.ReadAll(part) cte := part.Header.Get("Content-Transfer-Encoding") decoded := decodeBody(data, cte) + if strings.Contains(mediaType, "text/") { + decoded = decodeCharset(decoded, params["charset"]) + } // Check disposition for attachment disp := part.Header.Get("Content-Disposition") @@ -254,6 +260,24 @@ func parseMultipart(pm *ParsedMail, body io.Reader, boundary string) error { return nil } +// decodeCharset converts data from the declared MIME charset to UTF-8. +// Empty charset or "utf-8"/"us-ascii" are passed through unchanged. +func decodeCharset(data []byte, charset string) []byte { + charset = strings.ToLower(strings.TrimSpace(charset)) + if charset == "" || charset == "utf-8" || charset == "us-ascii" || charset == "ascii" { + return data + } + enc, err := htmlindex.Get(charset) + if err != nil { + return data + } + decoded, err := enc.NewDecoder().Bytes(data) + if err != nil { + return data + } + return decoded +} + // decodeBody decodes Content-Transfer-Encoding if needed. func decodeBody(data []byte, cte string) []byte { switch strings.ToLower(strings.TrimSpace(cte)) {