package storage import ( "context" "fmt" ) // ListNonASCIISubjects returns emails whose subject contains at least one // non-ASCII byte. // // Why a second lister next to ListRawEncodedSubjects: that one only matches the // RFC 2047 encoded-word pattern (`=?charset?B|Q?...?=`) and therefore cannot // find the second mojibake class at all — headers that were written with raw // 8-bit Windows-1252/ISO-8859-1 bytes and no encoded-word syntax. Those rows // contain no `=?...?=` marker; the only cheap SQL signal is "has bytes > 0x7F". // // The prefilter is `subject ~ '[^[:ascii:]]'`. Note that the obvious // alternative `octet_length(subject) <> length(subject)` is WRONG here: in a // SQL_ASCII database (which the production archive uses) length() counts bytes, // so that condition is never true and the query silently returns nothing. // The regex class works byte-wise in SQL_ASCII and char-wise in UTF8. // // The caller must narrow the result down with mailparser.NeedsCharsetRepair — // the vast majority of non-ASCII subjects are perfectly valid UTF-8 and must // not be touched. // // tenantID nil = all tenants. limit <= 0 = no limit. func (s *Store) ListNonASCIISubjects(ctx context.Context, tenantID *int64, limit int) ([]SubjectRow, error) { if s.db == nil { return nil, fmt.Errorf("storage: list non-ascii subjects: no database configured") } query := `SELECT id, tenant_id, COALESCE(subject, '') FROM emails WHERE subject IS NOT NULL AND subject ~ '[^[:ascii:]]'` args := []interface{}{} if tenantID != nil { args = append(args, *tenantID) query += fmt.Sprintf(" AND tenant_id = $%d", len(args)) } query += " ORDER BY received_at ASC" if limit > 0 { args = append(args, limit) query += fmt.Sprintf(" LIMIT $%d", len(args)) } rows, err := s.db.Query(ctx, query, args...) if err != nil { return nil, fmt.Errorf("storage: list non-ascii subjects: %w", err) } defer rows.Close() var out []SubjectRow for rows.Next() { var r SubjectRow if err := rows.Scan(&r.ID, &r.TenantID, &r.Subject); err != nil { return nil, fmt.Errorf("storage: list non-ascii subjects: scan: %w", err) } out = append(out, r) } if err := rows.Err(); err != nil { return nil, fmt.Errorf("storage: list non-ascii subjects: rows: %w", err) } return out, nil }