sysopsandClaude Sonnet 5
8f46d688a8
Getrennt von PROJ-84: Header (v.a. Subject) mit rohen 8-Bit-Bytes ohne RFC-2047-Encoded-Word-Syntax wurden nicht auf tatsächliches Charset geprüft, landeten als ungültiges UTF-8 in emails.subject und Manticore. Gleiche Lücke bei decodeCharset() für den Body ohne verwertbaren Content-Type. RepairUTF8/RepairUTF8Bytes (charset_repair.go): bytegenaue Reparatur, gültiges UTF-8 bleibt Identität, nur ungültige Byte-Sequenzen fallen auf Windows-1252 zurück. Attachment.Data bewusst ausgenommen (bleibt byte-exakt für Downloads). fix-subjects-Kommando erkennt jetzt beide Fälle (HasEncodedWord || NeedsCharsetRepair). Verifiziert auf 192.168.1.132: 54 zusätzliche Subject-Fälle, 219 Body-Fälle behoben (bodyInvalidUTF8 219 -> 0). --apply noch nicht ausgeführt, Body-Korrektur braucht zusätzlich reindex. Die ursprünglich gemeldete Amazon-Mail bleibt bewusst unverändert: Encoding-Fehler kam bereits so vom Absender (=3F statt =DC im Original-Encoded-Word), GoBD verbietet nachträgliche Korrektur archivierter Originalinhalte. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WapWkrQusDuBMhaN8WyuXB
67 lines
2.3 KiB
Go
67 lines
2.3 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
)
|
|
|
|
// ListNonASCIISubjects returns emails whose subject contains at least one
|
|
// non-ASCII byte.
|
|
//
|
|
// Why a second lister next to ListRawEncodedSubjects: that one only matches the
|
|
// RFC 2047 encoded-word pattern (`=?charset?B|Q?...?=`) and therefore cannot
|
|
// find the second mojibake class at all — headers that were written with raw
|
|
// 8-bit Windows-1252/ISO-8859-1 bytes and no encoded-word syntax. Those rows
|
|
// contain no `=?...?=` marker; the only cheap SQL signal is "has bytes > 0x7F".
|
|
//
|
|
// The prefilter is `subject ~ '[^[:ascii:]]'`. Note that the obvious
|
|
// alternative `octet_length(subject) <> length(subject)` is WRONG here: in a
|
|
// SQL_ASCII database (which the production archive uses) length() counts bytes,
|
|
// so that condition is never true and the query silently returns nothing.
|
|
// The regex class works byte-wise in SQL_ASCII and char-wise in UTF8.
|
|
//
|
|
// The caller must narrow the result down with mailparser.NeedsCharsetRepair —
|
|
// the vast majority of non-ASCII subjects are perfectly valid UTF-8 and must
|
|
// not be touched.
|
|
//
|
|
// tenantID nil = all tenants. limit <= 0 = no limit.
|
|
func (s *Store) ListNonASCIISubjects(ctx context.Context, tenantID *int64, limit int) ([]SubjectRow, error) {
|
|
if s.db == nil {
|
|
return nil, fmt.Errorf("storage: list non-ascii subjects: no database configured")
|
|
}
|
|
|
|
query := `SELECT id, tenant_id, COALESCE(subject, '')
|
|
FROM emails
|
|
WHERE subject IS NOT NULL
|
|
AND subject ~ '[^[:ascii:]]'`
|
|
args := []interface{}{}
|
|
if tenantID != nil {
|
|
args = append(args, *tenantID)
|
|
query += fmt.Sprintf(" AND tenant_id = $%d", len(args))
|
|
}
|
|
query += " ORDER BY received_at ASC"
|
|
if limit > 0 {
|
|
args = append(args, limit)
|
|
query += fmt.Sprintf(" LIMIT $%d", len(args))
|
|
}
|
|
|
|
rows, err := s.db.Query(ctx, query, args...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("storage: list non-ascii subjects: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
var out []SubjectRow
|
|
for rows.Next() {
|
|
var r SubjectRow
|
|
if err := rows.Scan(&r.ID, &r.TenantID, &r.Subject); err != nil {
|
|
return nil, fmt.Errorf("storage: list non-ascii subjects: scan: %w", err)
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, fmt.Errorf("storage: list non-ascii subjects: rows: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|