archivmail/pkg/mailparser/parser.go

package mailparser

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"mime/quotedprintable"
	"net/mail"
	"strings"
	"time"

	"golang.org/x/text/encoding/htmlindex"
)

// Attachment represents a MIME attachment in a parsed email.
type Attachment struct {
	Filename    string
	ContentType string
	Data        []byte
	Size        int
}

// ParsedMail holds the structured content of a parsed email message.
type ParsedMail struct {
	From        string
	To          []string
	CC          []string
	Subject     string
	MessageID   string
	InReplyTo   string   // In-Reply-To header (single message-id, no angle brackets)
	References  []string // References header (list of message-ids, no angle brackets)
	TextBody    string
	HTMLBody    string
	Date        time.Time
	Attachments []Attachment
	Raw         []byte

	// Accumulators used while walking MIME parts. Appending to the TextBody /
	// HTMLBody strings directly is O(n²) for mails with many parts, so parts
	// are collected here and joined once at the end of Parse.
	textBuf strings.Builder
	htmlBuf strings.Builder
}

// maxMultipartDepth caps MIME nesting. Malformed or malicious mails can nest
// multipart parts arbitrarily deep; without a limit the recursion in
// parseMultipart can exhaust the goroutine stack, which is an unrecoverable
// fatal error for the whole process.
const maxMultipartDepth = 20

// Parse parses a raw RFC 2822 / MIME email and returns a ParsedMail.
//
// Input is untrusted (IMAP/POP3/SMTP/upload). A panic inside the MIME/charset
// decoding stack must never take down the importer or the whole daemon, so it
// is converted into a normal parse error here.
func Parse(raw []byte) (pmOut *ParsedMail, errOut error) {
	defer func() {
		if r := recover(); r != nil {
			pmOut = nil
			errOut = fmt.Errorf("mailparser: panic while parsing message: %v", r)
		}
	}()

	msg, err := mail.ReadMessage(bytes.NewReader(raw))
	if err != nil {
		return nil, fmt.Errorf("mailparser: read message: %w", err)
	}

	pm := &ParsedMail{Raw: raw}

	// From
	if from := msg.Header.Get("From"); from != "" {
		addrs, err := mail.ParseAddressList(from)
		if err == nil && len(addrs) > 0 {
			pm.From = addrs[0].Address
		} else {
			pm.From = from
		}
	}

	// To
	if to := msg.Header.Get("To"); to != "" {
		addrs, err := mail.ParseAddressList(to)
		if err == nil {
			for _, a := range addrs {
				pm.To = append(pm.To, a.Address)
			}
		}
	}

	// CC
	if cc := msg.Header.Get("Cc"); cc != "" {
		addrs, err := mail.ParseAddressList(cc)
		if err == nil {
			for _, a := range addrs {
				pm.CC = append(pm.CC, a.Address)
			}
		}
	}

	// Subject - decode MIME encoded-words
	pm.Subject = decodeMIMEHeader(msg.Header.Get("Subject"))

	// Message-ID - strip angle brackets
	msgID := msg.Header.Get("Message-Id")
	pm.MessageID = strings.Trim(msgID, "<>")

	// In-Reply-To - strip angle brackets
	if irt := msg.Header.Get("In-Reply-To"); irt != "" {
		pm.InReplyTo = strings.Trim(strings.TrimSpace(irt), "<>")
	}

	// References - space-separated list of message-ids
	if refs := msg.Header.Get("References"); refs != "" {
		for _, r := range strings.Fields(refs) {
			r = strings.Trim(r, "<>")
			if r != "" {
				pm.References = append(pm.References, r)
			}
		}
	}

	// Date — try go-message parser first, then fallback formats, then zero
	if d, err := msg.Header.Date(); err == nil {
		pm.Date = d
	} else {
		// Some MUAs emit non-standard variants (e.g. "+0100 (CET)" suffix).
		// Try common RFC 2822 / non-standard formats before giving up.
		raw := strings.TrimSpace(msg.Header.Get("Date"))
		// Strip parenthesised timezone comment: "... +0100 (CET)" → "... +0100"
		if idx := strings.LastIndex(raw, "("); idx > 0 {
			raw = strings.TrimSpace(raw[:idx])
		}
		parsed := false
		for _, layout := range []string{
			"Mon, 2 Jan 2006 15:04:05 -0700",
			"Mon, 02 Jan 2006 15:04:05 -0700",
			"2 Jan 2006 15:04:05 -0700",
			"02 Jan 2006 15:04:05 -0700",
			"Mon, 2 Jan 2006 15:04:05 MST",
			"Mon, 02 Jan 2006 15:04:05 MST",
			// Colon in timezone offset (e.g. "+02:00") used by some MTA versions
			"Mon, 2 Jan 2006 15:04:05 -07:00",
			"Mon, 02 Jan 2006 15:04:05 -07:00",
			"2 Jan 2006 15:04:05 -07:00",
			"02 Jan 2006 15:04:05 -07:00",
			// Without seconds
			"Mon, 2 Jan 2006 15:04 -0700",
			"Mon, 02 Jan 2006 15:04 -0700",
			"2 Jan 2006 15:04 -0700",
			// Go stdlib aliases
			time.RFC1123Z,
			time.RFC1123,
		} {
			if t, err := time.Parse(layout, raw); err == nil {
				pm.Date = t
				parsed = true
				break
			}
		}
		if !parsed {
			// Some MTAs (e.g. PMG with German locale) use localised weekday names:
			// "So, 24 Aug 2025 00:05:17 +0200" instead of "Sun, 24 Aug 2025...".
			// Strip the "Weekday, " prefix (≤3 chars before the first comma) and retry.
			if commaIdx := strings.Index(raw, ","); commaIdx > 0 && commaIdx <= 3 {
				noWeekday := strings.TrimSpace(raw[commaIdx+1:])
				for _, layout := range []string{
					"2 Jan 2006 15:04:05 -0700",
					"02 Jan 2006 15:04:05 -0700",
					"2 Jan 2006 15:04:05 -07:00",
					"02 Jan 2006 15:04:05 -07:00",
					"2 Jan 2006 15:04:05 MST",
					"02 Jan 2006 15:04:05 MST",
					"2 Jan 2006 15:04 -0700",
					"02 Jan 2006 15:04 -0700",
				} {
					if t, err := time.Parse(layout, noWeekday); err == nil {
						pm.Date = t
						parsed = true
						break
					}
				}
			}
		}
		if !parsed {
			// Leave pm.Date as zero — storage will use DB DEFAULT NOW()
			pm.Date = time.Time{}
		}
	}

	// Parse body / MIME parts
	contentType := msg.Header.Get("Content-Type")
	mediaType, params, err := mime.ParseMediaType(contentType)
	if err != nil {
		// No content-type or parse error: treat as plain text
		body, _ := io.ReadAll(msg.Body)
		// Mails without any Content-Type are typically old 8-bit Latin-1
		// messages — repair them like a declared text/plain part.
		pm.TextBody = string(RepairUTF8Bytes(body))
		return pm, nil
	}

	if strings.HasPrefix(mediaType, "multipart/") {
		boundary := params["boundary"]
		// Ignore parse errors — return partial content instead of failing completely.
		// Malformed/truncated multipart emails still get metadata + whatever parts parsed.
		parseMultipart(pm, msg.Body, boundary, 0) //nolint:errcheck
		pm.TextBody = pm.textBuf.String()
		pm.HTMLBody = pm.htmlBuf.String()
	} else {
		body, _ := io.ReadAll(msg.Body)
		decoded := decodeBody(body, msg.Header.Get("Content-Transfer-Encoding"))
		if strings.HasPrefix(mediaType, "text/") || mediaType == "" {
			decoded = decodeCharset(decoded, params["charset"])
		}
		// The body always ends up as displayed/indexed text here, so the 8-bit
		// fallback repair applies to every branch (undeclared charset, wrongly
		// declared charset, or no usable Content-Type). No-op for valid UTF-8.
		if strings.Contains(mediaType, "html") {
			pm.HTMLBody = string(RepairUTF8Bytes(decoded))
		} else {
			pm.TextBody = string(RepairUTF8Bytes(decoded))
		}
	}

	return pm, nil
}

// parseMultipart walks MIME parts and fills text, html, and attachments.
// Truncated or malformed parts are skipped — partial content is better than nothing.
// depth guards against unbounded MIME nesting (stack overflow).
func parseMultipart(pm *ParsedMail, body io.Reader, boundary string, depth int) {
	if depth >= maxMultipartDepth || boundary == "" {
		return
	}
	mr := multipart.NewReader(body, boundary)
	for {
		part, err := mr.NextPart()
		if err != nil {
			// io.EOF = normal end; any other error = truncated/malformed — stop here.
			break
		}

		ct := part.Header.Get("Content-Type")
		mediaType, params, err := mime.ParseMediaType(ct)
		if err != nil {
			mediaType = "application/octet-stream"
			params = map[string]string{}
		}

		data, _ := io.ReadAll(part)
		cte := part.Header.Get("Content-Transfer-Encoding")
		decoded := decodeBody(data, cte)
		if strings.Contains(mediaType, "text/") {
			decoded = decodeCharset(decoded, params["charset"])
		}

		// Check disposition for attachment
		disp := part.Header.Get("Content-Disposition")
		dispType, dispParams, _ := mime.ParseMediaType(disp)
		filename := dispParams["filename"]
		if filename == "" {
			filename = params["name"]
		}
		filename = decodeMIMEHeader(filename)

		if strings.HasPrefix(dispType, "attachment") || filename != "" {
			pm.Attachments = append(pm.Attachments, Attachment{
				Filename:    filename,
				ContentType: mediaType,
				Data:        decoded,
				Size:        len(decoded),
			})
			continue
		}

		// Nested multipart
		if strings.HasPrefix(mediaType, "multipart/") {
			parseMultipart(pm, bytes.NewReader(decoded), params["boundary"], depth+1)
			continue
		}

		// Only the displayed/indexed body text gets the 8-bit fallback repair;
		// attachment bytes above stay byte-exact on purpose.
		switch {
		case strings.Contains(mediaType, "text/plain"):
			pm.textBuf.Write(RepairUTF8Bytes(decoded))
		case strings.Contains(mediaType, "text/html"):
			pm.htmlBuf.Write(RepairUTF8Bytes(decoded))
		}
	}
}

// decodeCharset converts data from the declared MIME charset to UTF-8.
// Empty charset or "utf-8"/"us-ascii" are passed through unchanged.
func decodeCharset(data []byte, charset string) []byte {
	charset = strings.ToLower(strings.TrimSpace(charset))
	if charset == "" || charset == "utf-8" || charset == "us-ascii" || charset == "ascii" {
		return data
	}
	enc, err := htmlindex.Get(charset)
	if err != nil {
		return data
	}
	decoded, err := enc.NewDecoder().Bytes(data)
	if err != nil {
		return data
	}
	return decoded
}

// decodeBody decodes Content-Transfer-Encoding if needed.
func decodeBody(data []byte, cte string) []byte {
	switch strings.ToLower(strings.TrimSpace(cte)) {
	case "quoted-printable":
		decoded, err := io.ReadAll(quotedprintable.NewReader(bytes.NewReader(data)))
		if err == nil {
			return decoded
		}
	case "base64":
		clean := bytes.ReplaceAll(data, []byte("\r\n"), []byte{})
		clean = bytes.ReplaceAll(clean, []byte("\n"), []byte{})
		clean = bytes.ReplaceAll(clean, []byte("\r"), []byte{})
		decoded := make([]byte, base64.StdEncoding.DecodedLen(len(clean)))
		n, err := base64.StdEncoding.Decode(decoded, clean)
		if err == nil {
			return decoded[:n]
		}
	}
	return data
}

// decodeMIMEHeader decodes RFC 2047 encoded-word headers. Go's mime package
// only knows UTF-8/US-ASCII/ISO-8859-1 natively; CharsetReader closes that
// gap for charsets like Windows-1252 using the same htmlindex lookup as
// decodeCharset for the body.
func decodeMIMEHeader(s string) string {
	dec := &mime.WordDecoder{
		CharsetReader: func(charset string, input io.Reader) (io.Reader, error) {
			data, err := io.ReadAll(input)
			if err != nil {
				return nil, err
			}
			return bytes.NewReader(decodeCharset(data, charset)), nil
		},
	}
	decoded, err := dec.DecodeHeader(s)
	if err != nil {
		// Undecodable encoded-word: keep the raw value, but still repair raw
		// 8-bit bytes so nothing invalid reaches the DB/index.
		return RepairUTF8(s)
	}
	// Headers without any encoded-word are returned byte-for-byte by
	// WordDecoder. Senders that write raw Windows-1252/ISO-8859-1 bytes into
	// the header would otherwise land as invalid UTF-8 in emails.subject.
	// RepairUTF8 is a no-op for valid UTF-8.
	return RepairUTF8(decoded)
}