archivdms/internal/pagesplit/pagesplit.go

// Package pagesplit implements barcode separator-page splitting for
// multi-page PDF ingest ("Trennseiten-Split", inspired by Paperless-ngx's
// ASN/separator barcode feature, adapted to archivdms's ingest pipeline).
//
// Idea: a scanner operator interleaves printed separator sheets carrying a
// well-known barcode (default value "ARCHIVDMS-SPLIT") between the individual
// receipts of one long scan run. At ingest the PDF is checked page by page for
// that barcode; where it is found, the document is cut, and the separator page
// itself is dropped (it is a control sheet, not content — same behaviour as
// Paperless-ngx). Each resulting part then runs through the completely normal
// staging path (own WORM file, own hash/duplicate check, own processing job).
//
// Design constraints this package follows, all inherited from the existing
// codebase:
//
//   - No CGO, no PDF library: everything is done by shelling out to the
//     poppler-utils binaries that are already a service dependency of the OCR
//     pipeline (pdfinfo, pdftoppm, pdfseparate, pdfunite) plus zbarimg via
//     internal/barcode. Deliberately NOT qpdf/pdftk — those would be a new
//     package dependency for something poppler already covers.
//   - Best-effort, fail-safe: every error path returns "no split" rather than
//     failing the upload. A scanner run that cannot be analysed must still be
//     archived, unsplit, rather than rejected. The one thing that is never
//     silently swallowed is a *partially* produced split — Split either yields
//     a complete set of parts or nothing at all.
//   - Off by default (Detector.Enabled), per the project's conservative rule
//     for new preprocessing behaviour (cf. the Otsu binarize switch in
//     internal/ocr).
//
// Scope note: only application/pdf is handled. Multi-page TIFF is a
// theoretically possible scanner output but is not currently produced by any
// archivdms ingest path (HTTP upload and the SFTP watcher both hand single
// images or PDFs to the pipeline), so it is intentionally out of scope here
// rather than half-supported.
package pagesplit

import (
	"bytes"
	"context"
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"log/slog"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"archivdms/internal/barcode"
)

// DefaultMarker is the barcode payload that marks a separator page when no
// other value is configured. Chosen to be unambiguous and unlikely to collide
// with a taxonomy barcode (internal/storage/taxonomy.go barcode_value) or with
// anything printed on a real invoice.
const DefaultMarker = "ARCHIVDMS-SPLIT"

// defaultRasterDPI is the resolution separator detection rasterizes at. Much
// lower than the 300 dpi the OCR pipeline uses: a separator sheet carries one
// large, high-contrast barcode, and 150 dpi decodes those reliably while
// keeping the extra pdftoppm pass cheap on long scan runs.
const defaultRasterDPI = 150

// defaultTimeout bounds each individual poppler subprocess call.
const defaultTimeout = 120 * time.Second

// defaultMaxPages caps how many pages are analysed. A scan run beyond this is
// treated as "not analysable" (no split, archived as one document) instead of
// spending unbounded time rasterizing — the same bounded-worst-case reasoning
// as internal/ocr's maxImagePixels clamp.
const defaultMaxPages = 200

// Detector performs separator-page detection and PDF splitting.
//
// Construct via New and set the optional fields afterwards; the zero value is
// disabled and therefore a safe no-op.
type Detector struct {
	// Enabled turns the whole feature on. False (zero value) => Split always
	// reports "no split".
	Enabled bool
	// Marker is the barcode payload identifying a separator page. Empty
	// defaults to DefaultMarker. Compared case-insensitively after trimming.
	Marker string
	// MarkerPrefix switches the comparison from "equals Marker" to "starts
	// with Marker", so operators can encode extra data on the separator sheet
	// (e.g. "ARCHIVDMS-SPLIT-2026-INVOICES") with a single configured value.
	MarkerPrefix bool
	// PdftoppmPath/PdfinfoPath/PdfseparatePath/PdfunitePath name the poppler
	// binaries. Empty values fall back to the plain command names.
	PdftoppmPath    string
	PdfinfoPath     string
	PdfseparatePath string
	PdfunitePath    string
	// TmpDir is the scratch base directory (config.StorageConfig.OCRTmpPath()).
	// Every Split call gets its own subdirectory, removed by Result.Cleanup.
	TmpDir string
	// RasterDPI overrides defaultRasterDPI. MaxPages overrides defaultMaxPages.
	RasterDPI int
	MaxPages  int
	// Timeout bounds each subprocess call. Zero => defaultTimeout.
	Timeout time.Duration
	// Logger receives best-effort diagnostics. Optional (nil = silent).
	Logger *slog.Logger
}

// New builds a Detector from the resolved config values.
func New(enabled bool, marker string, markerPrefix bool, pdftoppmPath, tmpDir string) *Detector {
	return &Detector{
		Enabled:      enabled,
		Marker:       marker,
		MarkerPrefix: markerPrefix,
		PdftoppmPath: pdftoppmPath,
		TmpDir:       tmpDir,
	}
}

// Result describes a completed split.
type Result struct {
	// Parts holds the absolute paths of the produced part PDFs, in original
	// page order. Always at least one entry when Split reports split == true.
	Parts []string
	// PartPageRanges[i] holds the 1-based [first,last] page numbers of Parts[i]
	// within the original document — audit-log material, so the aggregation of
	// pages into parts stays reconstructible after the original is gone.
	PartPageRanges [][2]int
	// SeparatorPages holds the 1-based page numbers that carried the marker
	// barcode and were therefore dropped.
	SeparatorPages []int
	// PageCount is the original document's total page count.
	PageCount int
	// Cleanup removes the scratch directory holding Parts. Never nil when
	// Split returned split == true; callers must defer it.
	Cleanup func()
}

// Split analyses pdfPath for separator pages and, if any are found, produces
// one part PDF per content segment.
//
// Returns split == false (with a nil Result) for every "carry on normally"
// outcome: detector disabled, poppler/zbarimg missing, fewer than two pages,
// page count above MaxPages, no separator barcode found, or every page being a
// separator page. Only genuinely unexpected failures return an error, and even
// those are meant to be treated by the caller as "archive unsplit" plus an
// audit entry — never as an upload failure.
//
// pdfPath must be a scratch/inbox file: it is only ever read, but the whole
// point of this function is that it runs BEFORE the file becomes a WORM
// archive object, so it must never be pointed at store/.
func (d *Detector) Split(ctx context.Context, pdfPath string) (res *Result, split bool, err error) {
	if d == nil || !d.Enabled {
		return nil, false, nil
	}
	for _, bin := range []string{d.pdfinfoPath(), d.pdftoppmPath(), d.pdfseparatePath(), d.pdfunitePath()} {
		if _, lookErr := exec.LookPath(bin); lookErr != nil {
			d.log(slog.LevelWarn, "pagesplit skipped: poppler binary not found in PATH",
				"binary", bin, "err", lookErr)
			return nil, false, nil
		}
	}
	if _, lookErr := exec.LookPath("zbarimg"); lookErr != nil {
		d.log(slog.LevelWarn, "pagesplit skipped: zbarimg not found in PATH", "err", lookErr)
		return nil, false, nil
	}

	pageCount, err := d.pageCount(ctx, pdfPath)
	if err != nil {
		return nil, false, fmt.Errorf("pagesplit: page count: %w", err)
	}
	if pageCount < 2 {
		return nil, false, nil
	}
	if pageCount > d.maxPages() {
		d.log(slog.LevelWarn, "pagesplit skipped: page count above limit",
			"file", pdfPath, "pages", pageCount, "max_pages", d.maxPages())
		return nil, false, nil
	}

	jobDir := filepath.Join(d.tmpDir(), "split-"+randomID())
	if mkErr := os.MkdirAll(jobDir, 0o750); mkErr != nil {
		return nil, false, fmt.Errorf("pagesplit: create scratch dir: %w", mkErr)
	}
	cleanup := func() { os.RemoveAll(jobDir) }
	// Anything below that returns without a successful split must not leak the
	// scratch directory; the success path hands cleanup to the caller instead.
	ok := false
	defer func() {
		if !ok {
			cleanup()
		}
	}()

	sepPages, err := d.detectSeparatorPages(ctx, pdfPath, jobDir, pageCount)
	if err != nil {
		return nil, false, fmt.Errorf("pagesplit: separator detection: %w", err)
	}
	if len(sepPages) == 0 {
		return nil, false, nil
	}

	ranges := contentRanges(pageCount, sepPages)
	if len(ranges) == 0 {
		// Pathological upload: only separator sheets, no content at all. Do not
		// silently discard it — archive the original unsplit so the operator
		// sees what was scanned.
		d.log(slog.LevelWarn, "pagesplit skipped: document consists of separator pages only",
			"file", pdfPath, "pages", pageCount)
		return nil, false, nil
	}

	partsDir := filepath.Join(jobDir, "parts")
	if mkErr := os.MkdirAll(partsDir, 0o750); mkErr != nil {
		return nil, false, fmt.Errorf("pagesplit: create parts dir: %w", mkErr)
	}
	var parts []string
	for i, rg := range ranges {
		partPath, perr := d.extractRange(ctx, pdfPath, partsDir, i+1, rg[0], rg[1])
		if perr != nil {
			// Partial split is never handed out — the caller falls back to
			// archiving the unsplit original.
			return nil, false, fmt.Errorf("pagesplit: extract pages %d-%d: %w", rg[0], rg[1], perr)
		}
		parts = append(parts, partPath)
	}

	d.log(slog.LevelInfo, "pagesplit produced parts",
		"file", pdfPath, "pages", pageCount, "separator_pages", sepPages, "parts", len(parts))

	ok = true
	return &Result{
		Parts:          parts,
		PartPageRanges: ranges,
		SeparatorPages: sepPages,
		PageCount:      pageCount,
		Cleanup:        cleanup,
	}, true, nil
}

// IsSeparatorValue reports whether a decoded barcode payload marks a separator
// page under this detector's marker configuration.
func (d *Detector) IsSeparatorValue(value string) bool {
	v := strings.ToUpper(strings.TrimSpace(value))
	m := strings.ToUpper(strings.TrimSpace(d.marker()))
	if v == "" || m == "" {
		return false
	}
	if d.MarkerPrefix {
		return strings.HasPrefix(v, m)
	}
	return v == m
}

var pdfinfoPagesRegex = regexp.MustCompile(`(?m)^Pages:\s+(\d+)`)

// pageCount reads the page count via `pdfinfo`.
func (d *Detector) pageCount(ctx context.Context, pdfPath string) (int, error) {
	cctx, cancel := context.WithTimeout(ctx, d.timeout())
	defer cancel()

	cmd := exec.CommandContext(cctx, d.pdfinfoPath(), pdfPath)
	var out, stderr bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		return 0, fmt.Errorf("pdfinfo failed: %w (%s)", err, strings.TrimSpace(stderr.String()))
	}
	m := pdfinfoPagesRegex.FindStringSubmatch(out.String())
	if m == nil {
		return 0, fmt.Errorf("pdfinfo output had no Pages line")
	}
	n, err := strconv.Atoi(m[1])
	if err != nil {
		return 0, fmt.Errorf("pdfinfo page count unparseable: %w", err)
	}
	return n, nil
}

// pageNumRegex pulls the page number out of the filenames pdftoppm/pdfseparate
// generate (page-01.png, page-1.png, seg-12.pdf, ...). Sorting on that number
// rather than lexically matters as soon as a run crosses 9 or 99 pages.
var pageNumRegex = regexp.MustCompile(`(\d+)\D*$`)

// detectSeparatorPages rasterizes every page once and decodes barcodes on it,
// returning the 1-based page numbers that carry the marker.
func (d *Detector) detectSeparatorPages(ctx context.Context, pdfPath, jobDir string, pageCount int) ([]int, error) {
	rasterDir := filepath.Join(jobDir, "raster")
	if err := os.MkdirAll(rasterDir, 0o750); err != nil {
		return nil, fmt.Errorf("create raster dir: %w", err)
	}

	cctx, cancel := context.WithTimeout(ctx, d.timeout())
	defer cancel()

	prefix := filepath.Join(rasterDir, "page")
	cmd := exec.CommandContext(cctx, d.pdftoppmPath(),
		"-r", strconv.Itoa(d.rasterDPI()), "-png", pdfPath, prefix)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		return nil, fmt.Errorf("pdftoppm failed: %w (%s)", err, strings.TrimSpace(stderr.String()))
	}

	pages, err := sortedNumberedFiles(rasterDir, ".png")
	if err != nil {
		return nil, err
	}
	if len(pages) != pageCount {
		// Mismatch means the page-number mapping below cannot be trusted, and a
		// wrong mapping would cut the document in the wrong place — refuse.
		return nil, fmt.Errorf("rasterized %d pages but pdfinfo reported %d", len(pages), pageCount)
	}

	var sep []int
	for i, page := range pages {
		codes, decErr := barcode.DecodeBarcodes(ctx, page)
		if decErr != nil {
			// Best-effort per page, exactly as in internal/ocr: a page whose
			// barcode pass errored is simply treated as a content page.
			continue
		}
		for _, code := range codes {
			if d.IsSeparatorValue(code) {
				sep = append(sep, i+1)
				break
			}
		}
	}
	return sep, nil
}

// contentRanges turns a page count plus the separator page numbers into the
// 1-based inclusive page ranges of the content segments, dropping the
// separator pages themselves and any empty segment (two adjacent separator
// sheets, or one at the very start/end).
func contentRanges(pageCount int, sepPages []int) [][2]int {
	isSep := make(map[int]bool, len(sepPages))
	for _, p := range sepPages {
		isSep[p] = true
	}
	var ranges [][2]int
	start := 0
	for p := 1; p <= pageCount; p++ {
		if isSep[p] {
			if start != 0 {
				ranges = append(ranges, [2]int{start, p - 1})
				start = 0
			}
			continue
		}
		if start == 0 {
			start = p
		}
	}
	if start != 0 {
		ranges = append(ranges, [2]int{start, pageCount})
	}
	return ranges
}

// extractRange writes pages [first,last] of pdfPath into one PDF under
// partsDir, using pdfseparate (per-page extraction) plus pdfunite (re-merge)
// — the poppler-only equivalent of `qpdf --pages`.
func (d *Detector) extractRange(ctx context.Context, pdfPath, partsDir string, index, first, last int) (string, error) {
	segDir := filepath.Join(partsDir, fmt.Sprintf("seg-%03d", index))
	if err := os.MkdirAll(segDir, 0o750); err != nil {
		return "", fmt.Errorf("create segment dir: %w", err)
	}

	sepCtx, cancelSep := context.WithTimeout(ctx, d.timeout())
	defer cancelSep()

	pattern := filepath.Join(segDir, "p-%d.pdf")
	cmd := exec.CommandContext(sepCtx, d.pdfseparatePath(),
		"-f", strconv.Itoa(first), "-l", strconv.Itoa(last), pdfPath, pattern)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		return "", fmt.Errorf("pdfseparate failed: %w (%s)", err, strings.TrimSpace(stderr.String()))
	}

	pageFiles, err := sortedNumberedFiles(segDir, ".pdf")
	if err != nil {
		return "", err
	}
	want := last - first + 1
	if len(pageFiles) != want {
		return "", fmt.Errorf("pdfseparate produced %d pages, expected %d", len(pageFiles), want)
	}
	if len(pageFiles) == 1 {
		// Single-page segment: the extracted page already IS the part.
		return pageFiles[0], nil
	}

	uniteCtx, cancelUnite := context.WithTimeout(ctx, d.timeout())
	defer cancelUnite()

	outPath := filepath.Join(partsDir, fmt.Sprintf("part-%03d.pdf", index))
	args := append(append([]string{}, pageFiles...), outPath)
	uniteCmd := exec.CommandContext(uniteCtx, d.pdfunitePath(), args...)
	var uniteErr bytes.Buffer
	uniteCmd.Stderr = &uniteErr
	if err := uniteCmd.Run(); err != nil {
		os.Remove(outPath)
		return "", fmt.Errorf("pdfunite failed: %w (%s)", err, strings.TrimSpace(uniteErr.String()))
	}
	if fi, statErr := os.Stat(outPath); statErr != nil || fi.Size() == 0 {
		os.Remove(outPath)
		return "", fmt.Errorf("pdfunite produced empty/missing output: %v", statErr)
	}
	return outPath, nil
}

// sortedNumberedFiles lists dir's files with the given extension, sorted by
// the trailing number in their name (numeric, not lexical).
func sortedNumberedFiles(dir, ext string) ([]string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, fmt.Errorf("read dir %s: %w", dir, err)
	}
	type numbered struct {
		path string
		num  int
	}
	var found []numbered
	for _, entry := range entries {
		if entry.IsDir() || !strings.HasSuffix(entry.Name(), ext) {
			continue
		}
		num := 0
		base := strings.TrimSuffix(entry.Name(), ext)
		if m := pageNumRegex.FindStringSubmatch(base); m != nil {
			num, _ = strconv.Atoi(m[1])
		}
		found = append(found, numbered{path: filepath.Join(dir, entry.Name()), num: num})
	}
	sort.Slice(found, func(i, j int) bool {
		if found[i].num != found[j].num {
			return found[i].num < found[j].num
		}
		return found[i].path < found[j].path
	})
	paths := make([]string, 0, len(found))
	for _, f := range found {
		paths = append(paths, f.path)
	}
	return paths, nil
}

// randomID returns a random hex string for scratch directory names. Kept
// dependency-free, same approach as internal/ocr.randomID.
func randomID() string {
	b := make([]byte, 16)
	if _, err := rand.Read(b); err != nil {
		return fmt.Sprintf("job-%d", time.Now().UnixNano())
	}
	return hex.EncodeToString(b)
}

func (d *Detector) log(level slog.Level, msg string, args ...any) {
	if d == nil || d.Logger == nil {
		return
	}
	d.Logger.Log(context.Background(), level, msg, args...)
}

func (d *Detector) marker() string {
	if strings.TrimSpace(d.Marker) == "" {
		return DefaultMarker
	}
	return d.Marker
}

func (d *Detector) pdftoppmPath() string    { return orDefault(d.PdftoppmPath, "pdftoppm") }
func (d *Detector) pdfinfoPath() string     { return orDefault(d.PdfinfoPath, "pdfinfo") }
func (d *Detector) pdfseparatePath() string { return orDefault(d.PdfseparatePath, "pdfseparate") }
func (d *Detector) pdfunitePath() string    { return orDefault(d.PdfunitePath, "pdfunite") }

func orDefault(v, def string) string {
	if strings.TrimSpace(v) == "" {
		return def
	}
	return v
}

func (d *Detector) tmpDir() string {
	if strings.TrimSpace(d.TmpDir) == "" {
		return os.TempDir()
	}
	return d.TmpDir
}

func (d *Detector) rasterDPI() int {
	if d.RasterDPI <= 0 {
		return defaultRasterDPI
	}
	return d.RasterDPI
}

func (d *Detector) maxPages() int {
	if d.MaxPages <= 0 {
		return defaultMaxPages
	}
	return d.MaxPages
}

func (d *Detector) timeout() time.Duration {
	if d.Timeout <= 0 {
		return defaultTimeout
	}
	return d.Timeout
}