package ocr import ( "bytes" "context" "encoding/base64" "fmt" "html" "io" "mime" "log/slog" "mime/multipart" "mime/quotedprintable" "net/mail" "os" "os/exec" "path/filepath" "regexp" "strings" "time" ) // officeMimeTypes are the document formats we route through LibreOffice // (soffice --headless --convert-to pdf) before OCR. Mirrors the Paperless-ngx // Gotenberg / Docspell LibreOffice conversion stage: the resulting PDF is then // fed to the normal pdftotext / pdftoppm+tesseract pipeline (ocrPDF), so both // text-layer PDFs and scanned-image content inside the office file are covered. var officeMimeTypes = map[string]bool{ // Word / text processing "application/msword": true, "application/vnd.openxmlformats-officedocument.wordprocessingml.document": true, "application/vnd.oasis.opendocument.text": true, "application/rtf": true, "text/rtf": true, // Spreadsheets "application/vnd.ms-excel": true, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true, "application/vnd.oasis.opendocument.spreadsheet": true, // Presentations "application/vnd.ms-powerpoint": true, "application/vnd.openxmlformats-officedocument.presentationml.presentation": true, "application/vnd.oasis.opendocument.presentation": true, } // isOfficeMime reports whether mimeType names an Office document format that // Extract routes through LibreOffice conversion. func isOfficeMime(mimeType string) bool { return officeMimeTypes[strings.ToLower(strings.TrimSpace(mimeType))] } // officeConvertTimeout bounds a single LibreOffice conversion. Deliberately // more generous than the per-OCR-call timeout: a cold soffice start plus a // large spreadsheet can legitimately take longer than a tesseract page. func (e *Extractor) officeConvertTimeout() time.Duration { base := e.timeout() if base < 120*time.Second { return 120 * time.Second } return base } // officeToPDF converts an Office document at filePath to a temporary PDF via // LibreOffice headless, returning the PDF path and a cleanup func the caller // must defer. LibreOffice needs a private user-profile dir to run reliably and // concurrently (multiple soffice instances sharing the default profile clash), // so each conversion gets its own scratch dir under TmpDir. func (e *Extractor) officeToPDF(ctx context.Context, filePath string) (string, func(), error) { bin := e.sofficePath() if _, err := exec.LookPath(bin); err != nil { return "", nil, fmt.Errorf("ocr: libreoffice (%s) not found in PATH: %w", bin, err) } workDir := filepath.Join(e.tmpDir(), "office-"+randomID()) if err := os.MkdirAll(workDir, 0o700); err != nil { return "", nil, fmt.Errorf("ocr: create office convert dir: %w", err) } cleanup := func() { os.RemoveAll(workDir) } profileDir := filepath.Join(workDir, "profile") cctx, cancel := context.WithTimeout(ctx, e.officeConvertTimeout()) defer cancel() args := []string{ "--headless", "--norestore", "--nologo", "--nolockcheck", "-env:UserInstallation=file://" + profileDir, "--convert-to", "pdf", "--outdir", workDir, filePath, } cmd := exec.CommandContext(cctx, bin, args...) var stderr bytes.Buffer cmd.Stderr = &stderr if err := cmd.Run(); err != nil { cleanup() if cctx.Err() == context.DeadlineExceeded { return "", nil, fmt.Errorf("ocr: libreoffice conversion timed out after %s", e.officeConvertTimeout()) } return "", nil, fmt.Errorf("ocr: libreoffice conversion failed: %w (%s)", err, strings.TrimSpace(stderr.String())) } // soffice names the output .pdf in --outdir. Prefer that // exact name, but fall back to the first *.pdf in the dir in case the base // name was sanitized. base := strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath)) pdfPath := filepath.Join(workDir, base+".pdf") if _, err := os.Stat(pdfPath); err != nil { matches, _ := filepath.Glob(filepath.Join(workDir, "*.pdf")) if len(matches) == 0 { cleanup() return "", nil, fmt.Errorf("ocr: libreoffice produced no pdf for %s", filepath.Base(filePath)) } pdfPath = matches[0] } e.log(slog.LevelInfo, "office document converted to pdf", "src", filepath.Base(filePath), "pdf", filepath.Base(pdfPath)) return pdfPath, cleanup, nil } // extractEML parses an .eml file and returns its human-readable text: a short // header block (Date/From/To/Cc/Subject, MIME-word-decoded) followed by the // concatenated text of every text/plain and (tag-stripped) text/html body part. // Binary attachments are ignored. Best-effort throughout — a malformed message // yields whatever could be parsed rather than an error, so an e-mail is never // silently dropped from full-text search. func (e *Extractor) extractEML(filePath string) (*Result, error) { data, err := os.ReadFile(filePath) if err != nil { return nil, fmt.Errorf("ocr: read eml: %w", err) } msg, err := mail.ReadMessage(bytes.NewReader(data)) if err != nil { return nil, fmt.Errorf("ocr: parse eml: %w", err) } var sb strings.Builder dec := new(mime.WordDecoder) for _, h := range []string{"Date", "From", "To", "Cc", "Subject"} { v := msg.Header.Get(h) if v == "" { continue } if d, derr := dec.DecodeHeader(v); derr == nil { v = d } sb.WriteString(h) sb.WriteString(": ") sb.WriteString(v) sb.WriteString("\n") } sb.WriteString("\n") body := mailPartText(msg.Header.Get("Content-Type"), msg.Header.Get("Content-Transfer-Encoding"), msg.Body) sb.WriteString(body) return &Result{Text: strings.TrimSpace(sb.String())}, nil } // mailPartText recursively extracts readable text from a MIME part. multipart/* // containers are walked; text/plain is decoded verbatim, text/html is decoded // and tag-stripped; everything else (attachments, images) is skipped. func mailPartText(contentType, cte string, body io.Reader) string { mediaType, params, err := mime.ParseMediaType(contentType) if err != nil || mediaType == "" { // No/invalid Content-Type: assume text/plain. return decodeMailBody(body, cte) } if strings.HasPrefix(mediaType, "multipart/") { boundary := params["boundary"] if boundary == "" { return "" } mr := multipart.NewReader(body, boundary) var parts []string for { p, perr := mr.NextPart() if perr != nil { break } pt := mailPartText(p.Header.Get("Content-Type"), p.Header.Get("Content-Transfer-Encoding"), p) p.Close() if strings.TrimSpace(pt) != "" { parts = append(parts, pt) } } return strings.Join(parts, "\n") } switch { case strings.HasPrefix(mediaType, "text/plain"): return decodeMailBody(body, cte) case strings.HasPrefix(mediaType, "text/html"): return stripHTML(decodeMailBody(body, cte)) default: return "" // attachment / binary part } } // decodeMailBody reads a leaf MIME part, undoing base64 / quoted-printable // transfer encoding. func decodeMailBody(r io.Reader, cte string) string { switch strings.ToLower(strings.TrimSpace(cte)) { case "base64": r = base64.NewDecoder(base64.StdEncoding, r) case "quoted-printable": r = quotedprintable.NewReader(r) } b, err := io.ReadAll(r) if err != nil { return string(b) // return whatever decoded before the error } return string(b) } var ( htmlTagRe = regexp.MustCompile(`(?s)<(script|style)[^>]*>.*?`) anyTagRe = regexp.MustCompile(`(?s)<[^>]*>`) wsRe = regexp.MustCompile(`[ \t\f\v]+`) blankRe = regexp.MustCompile(`\n{3,}`) ) // stripHTML reduces an HTML body to readable plain text: drops script/style // blocks and all tags, unescapes entities, and collapses runaway whitespace. func stripHTML(s string) string { s = htmlTagRe.ReplaceAllString(s, " ") s = anyTagRe.ReplaceAllString(s, " ") s = html.UnescapeString(s) s = wsRe.ReplaceAllString(s, " ") s = blankRe.ReplaceAllString(s, "\n\n") return strings.TrimSpace(s) }