archivdms/internal/ocr/convert.go

package ocr

import (
	"bytes"
	"context"
	"encoding/base64"
	"fmt"
	"html"
	"io"
	"mime"
	"log/slog"
	"mime/multipart"
	"mime/quotedprintable"
	"net/mail"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"
)

// officeMimeTypes are the document formats we route through LibreOffice
// (soffice --headless --convert-to pdf) before OCR. Mirrors the Paperless-ngx
// Gotenberg / Docspell LibreOffice conversion stage: the resulting PDF is then
// fed to the normal pdftotext / pdftoppm+tesseract pipeline (ocrPDF), so both
// text-layer PDFs and scanned-image content inside the office file are covered.
var officeMimeTypes = map[string]bool{
	// Word / text processing
	"application/msword": true,
	"application/vnd.openxmlformats-officedocument.wordprocessingml.document": true,
	"application/vnd.oasis.opendocument.text":                                 true,
	"application/rtf":                                                         true,
	"text/rtf":                                                                true,
	// Spreadsheets
	"application/vnd.ms-excel": true,
	"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true,
	"application/vnd.oasis.opendocument.spreadsheet":                    true,
	// Presentations
	"application/vnd.ms-powerpoint": true,
	"application/vnd.openxmlformats-officedocument.presentationml.presentation": true,
	"application/vnd.oasis.opendocument.presentation":                          true,
}

// isOfficeMime reports whether mimeType names an Office document format that
// Extract routes through LibreOffice conversion.
func isOfficeMime(mimeType string) bool {
	return officeMimeTypes[strings.ToLower(strings.TrimSpace(mimeType))]
}

// officeConvertTimeout bounds a single LibreOffice conversion. Deliberately
// more generous than the per-OCR-call timeout: a cold soffice start plus a
// large spreadsheet can legitimately take longer than a tesseract page.
func (e *Extractor) officeConvertTimeout() time.Duration {
	base := e.timeout()
	if base < 120*time.Second {
		return 120 * time.Second
	}
	return base
}

// officeToPDF converts an Office document at filePath to a temporary PDF via
// LibreOffice headless, returning the PDF path and a cleanup func the caller
// must defer. LibreOffice needs a private user-profile dir to run reliably and
// concurrently (multiple soffice instances sharing the default profile clash),
// so each conversion gets its own scratch dir under TmpDir.
func (e *Extractor) officeToPDF(ctx context.Context, filePath string) (string, func(), error) {
	bin := e.sofficePath()
	if _, err := exec.LookPath(bin); err != nil {
		return "", nil, fmt.Errorf("ocr: libreoffice (%s) not found in PATH: %w", bin, err)
	}

	workDir := filepath.Join(e.tmpDir(), "office-"+randomID())
	if err := os.MkdirAll(workDir, 0o700); err != nil {
		return "", nil, fmt.Errorf("ocr: create office convert dir: %w", err)
	}
	cleanup := func() { os.RemoveAll(workDir) }

	profileDir := filepath.Join(workDir, "profile")

	cctx, cancel := context.WithTimeout(ctx, e.officeConvertTimeout())
	defer cancel()

	args := []string{
		"--headless", "--norestore", "--nologo", "--nolockcheck",
		"-env:UserInstallation=file://" + profileDir,
		"--convert-to", "pdf", "--outdir", workDir, filePath,
	}
	cmd := exec.CommandContext(cctx, bin, args...)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		cleanup()
		if cctx.Err() == context.DeadlineExceeded {
			return "", nil, fmt.Errorf("ocr: libreoffice conversion timed out after %s", e.officeConvertTimeout())
		}
		return "", nil, fmt.Errorf("ocr: libreoffice conversion failed: %w (%s)", err, strings.TrimSpace(stderr.String()))
	}

	// soffice names the output <input-basename>.pdf in --outdir. Prefer that
	// exact name, but fall back to the first *.pdf in the dir in case the base
	// name was sanitized.
	base := strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath))
	pdfPath := filepath.Join(workDir, base+".pdf")
	if _, err := os.Stat(pdfPath); err != nil {
		matches, _ := filepath.Glob(filepath.Join(workDir, "*.pdf"))
		if len(matches) == 0 {
			cleanup()
			return "", nil, fmt.Errorf("ocr: libreoffice produced no pdf for %s", filepath.Base(filePath))
		}
		pdfPath = matches[0]
	}
	e.log(slog.LevelInfo, "office document converted to pdf",
		"src", filepath.Base(filePath), "pdf", filepath.Base(pdfPath))
	return pdfPath, cleanup, nil
}

// extractEML parses an .eml file and returns its human-readable text: a short
// header block (Date/From/To/Cc/Subject, MIME-word-decoded) followed by the
// concatenated text of every text/plain and (tag-stripped) text/html body part.
// Binary attachments are ignored. Best-effort throughout — a malformed message
// yields whatever could be parsed rather than an error, so an e-mail is never
// silently dropped from full-text search.
func (e *Extractor) extractEML(filePath string) (*Result, error) {
	data, err := os.ReadFile(filePath)
	if err != nil {
		return nil, fmt.Errorf("ocr: read eml: %w", err)
	}
	msg, err := mail.ReadMessage(bytes.NewReader(data))
	if err != nil {
		return nil, fmt.Errorf("ocr: parse eml: %w", err)
	}

	var sb strings.Builder
	dec := new(mime.WordDecoder)
	for _, h := range []string{"Date", "From", "To", "Cc", "Subject"} {
		v := msg.Header.Get(h)
		if v == "" {
			continue
		}
		if d, derr := dec.DecodeHeader(v); derr == nil {
			v = d
		}
		sb.WriteString(h)
		sb.WriteString(": ")
		sb.WriteString(v)
		sb.WriteString("\n")
	}
	sb.WriteString("\n")

	body := mailPartText(msg.Header.Get("Content-Type"), msg.Header.Get("Content-Transfer-Encoding"), msg.Body)
	sb.WriteString(body)

	return &Result{Text: strings.TrimSpace(sb.String())}, nil
}

// mailPartText recursively extracts readable text from a MIME part. multipart/*
// containers are walked; text/plain is decoded verbatim, text/html is decoded
// and tag-stripped; everything else (attachments, images) is skipped.
func mailPartText(contentType, cte string, body io.Reader) string {
	mediaType, params, err := mime.ParseMediaType(contentType)
	if err != nil || mediaType == "" {
		// No/invalid Content-Type: assume text/plain.
		return decodeMailBody(body, cte)
	}

	if strings.HasPrefix(mediaType, "multipart/") {
		boundary := params["boundary"]
		if boundary == "" {
			return ""
		}
		mr := multipart.NewReader(body, boundary)
		var parts []string
		for {
			p, perr := mr.NextPart()
			if perr != nil {
				break
			}
			pt := mailPartText(p.Header.Get("Content-Type"), p.Header.Get("Content-Transfer-Encoding"), p)
			p.Close()
			if strings.TrimSpace(pt) != "" {
				parts = append(parts, pt)
			}
		}
		return strings.Join(parts, "\n")
	}

	switch {
	case strings.HasPrefix(mediaType, "text/plain"):
		return decodeMailBody(body, cte)
	case strings.HasPrefix(mediaType, "text/html"):
		return stripHTML(decodeMailBody(body, cte))
	default:
		return "" // attachment / binary part
	}
}

// decodeMailBody reads a leaf MIME part, undoing base64 / quoted-printable
// transfer encoding.
func decodeMailBody(r io.Reader, cte string) string {
	switch strings.ToLower(strings.TrimSpace(cte)) {
	case "base64":
		r = base64.NewDecoder(base64.StdEncoding, r)
	case "quoted-printable":
		r = quotedprintable.NewReader(r)
	}
	b, err := io.ReadAll(r)
	if err != nil {
		return string(b) // return whatever decoded before the error
	}
	return string(b)
}

var (
	htmlTagRe = regexp.MustCompile(`(?s)<(script|style)[^>]*>.*?</(script|style)>`)
	anyTagRe  = regexp.MustCompile(`(?s)<[^>]*>`)
	wsRe      = regexp.MustCompile(`[ \t\f\v]+`)
	blankRe   = regexp.MustCompile(`\n{3,}`)
)

// stripHTML reduces an HTML body to readable plain text: drops script/style
// blocks and all tags, unescapes entities, and collapses runaway whitespace.
func stripHTML(s string) string {
	s = htmlTagRe.ReplaceAllString(s, " ")
	s = anyTagRe.ReplaceAllString(s, " ")
	s = html.UnescapeString(s)
	s = wsRe.ReplaceAllString(s, " ")
	s = blankRe.ReplaceAllString(s, "\n\n")
	return strings.TrimSpace(s)
}