archivdms/internal/api/date_extraction.go

package api

import (
	"regexp"
	"strconv"
	"strings"
	"time"
)

// extractDocumentDate scans OCR text for the most plausible document/invoice
// date and returns it, or nil if none is found. It is a deliberately simple,
// rule-based (no NLP, no external library, CGO-free) heuristic in the same
// spirit as titleFromOCRText.
//
// Recognised formats:
//   - DD.MM.YYYY   (German, e.g. 31.12.2024)
//   - DD.MM.YY     (German short year, e.g. 31.12.24 -> 2024)
//   - DD/MM/YYYY   (slash variant, incl. DD/MM/YY)
//   - YYYY-MM-DD   (ISO 8601, e.g. 2024-12-31)
//   - "15. März 2026" / "15. Mär. 2026" (spelled-out German month names)
//
// Plausibility: month 1-12, day 1-31 with an explicit calendar check (time.Date
// would normalise an impossible day, so we reject e.g. "31.02." rather than
// silently shifting it to March), year not before dateMinYear, and never a date
// in the future beyond today + dateFutureToleranceDays (a small tolerance for
// timezone/clock skew). These filters drop copyright years, footer years and
// stray digit runs.
//
// Scoring (see scoreForDatePosition): each candidate gets a confidence based on
// signal words in a small text window around it ("Rechnungsdatum", "vom", ...).
// The candidate with the highest score wins; on a tie the earliest occurrence in
// the text wins (document head = usually the issue date). Callers that only need
// the date use this function; callers that also need the confidence use
// extractDocumentDateWithScore.
const (
	dateMinYear               = 1990
	dateFutureToleranceDays   = 2
	dateWindowRadius          = 40
	dateScoreNoContext        = 0.4
)

// dateKeyword pairs a lowercase signal word with the confidence a date near it
// receives. Ordered by descending score: scoreForDatePosition returns the score
// of the first (=highest) keyword found in the window. Kept short and flat on
// purpose — no weighting engine, GoBD-traceable.
type dateKeyword struct {
	word  string
	score float64
}

var dateKeywords = []dateKeyword{
	{"rechnungsdatum", 0.9},
	{"belegdatum", 0.9},
	{"ausstellungsdatum", 0.9},
	{"rechnung vom", 0.9},
	{"beleg vom", 0.9},
	{"datum", 0.75},
	{"vom", 0.55},
}

// dateGermanMonths maps lowercased German month names and common abbreviations
// (with the trailing dot already stripped) to their month number.
var dateGermanMonths = map[string]int{
	"januar": 1, "jan": 1,
	"februar": 2, "feb": 2,
	"märz": 3, "maerz": 3, "mär": 3, "mrz": 3,
	"april": 4, "apr": 4,
	"mai": 5,
	"juni": 6, "jun": 6,
	"juli": 7, "jul": 7,
	"august": 8, "aug": 8,
	"september": 9, "sep": 9, "sept": 9,
	"oktober": 10, "okt": 10,
	"november": 11, "nov": 11,
	"dezember": 12, "dez": 12,
}

// dateCandidateRe matches every supported format in a single alternation. Named
// groups keep the branch handling readable. Word boundaries avoid gluing onto
// surrounding digits (e.g. a phone number). Case-insensitive for month names.
var dateCandidateRe = regexp.MustCompile(
	`(?i)(?:\b(?P<gd>\d{1,2})\.(?P<gm>\d{1,2})\.(?P<gy>\d{4}|\d{2})\b)` +
		`|(?:\b(?P<sd>\d{1,2})/(?P<sm>\d{1,2})/(?P<sy>\d{4}|\d{2})\b)` +
		`|(?:\b(?P<iy>\d{4})-(?P<im>\d{1,2})-(?P<id>\d{1,2})\b)` +
		`|(?:\b(?P<td>\d{1,2})\.?\s+(?P<tmon>[A-Za-zäöüÄÖÜ]+)\.?\s+(?P<ty>\d{4})\b)`,
)

// sameDate reports whether two optional dates refer to the same calendar day
// (or are both nil). Used by reprocess to skip a no-op document_date update.
func sameDate(a, b *time.Time) bool {
	if a == nil || b == nil {
		return a == nil && b == nil
	}
	ay, am, ad := a.Date()
	by, bm, bd := b.Date()
	return ay == by && am == bm && ad == bd
}

// scoreForDatePosition returns the confidence for a date match found at byte
// offset start in text, based on signal words within dateWindowRadius chars.
func scoreForDatePosition(lowerText string, start int) float64 {
	lo := start - dateWindowRadius
	if lo < 0 {
		lo = 0
	}
	hi := start + dateWindowRadius
	if hi > len(lowerText) {
		hi = len(lowerText)
	}
	window := lowerText[lo:hi]
	for _, kw := range dateKeywords {
		if strings.Contains(window, kw.word) {
			return kw.score
		}
	}
	return dateScoreNoContext
}

// parseDateMatch turns one regex submatch into a validated calendar date, or
// returns ok=false if the match is implausible.
func parseDateMatch(names, m []string) (time.Time, bool) {
	now := time.Now()
	maxDate := now.AddDate(0, 0, dateFutureToleranceDays)
	var day, month, year int
	var monthName string
	for i, name := range names {
		if m[i] == "" {
			continue
		}
		switch name {
		case "gd", "sd", "id", "td":
			day, _ = strconv.Atoi(m[i])
		case "gm", "sm", "im":
			month, _ = strconv.Atoi(m[i])
		case "gy", "sy":
			y, _ := strconv.Atoi(m[i])
			if len(m[i]) == 2 {
				// Two-digit year: interpret as 2000-2099. Anything above the
				// future tolerance is rejected below.
				y += 2000
			}
			year = y
		case "iy", "ty":
			year, _ = strconv.Atoi(m[i])
		case "tmon":
			monthName = m[i]
		}
	}
	if monthName != "" {
		mn, ok := dateGermanMonths[strings.ToLower(monthName)]
		if !ok {
			return time.Time{}, false
		}
		month = mn
	}
	if year < dateMinYear {
		return time.Time{}, false
	}
	if month < 1 || month > 12 {
		return time.Time{}, false
	}
	if day < 1 || day > 31 {
		return time.Time{}, false
	}
	d := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC)
	// Reject normalised-away impossible days (e.g. 31.02. -> 03.03.).
	if d.Day() != day || int(d.Month()) != month || d.Year() != year {
		return time.Time{}, false
	}
	// No future dates beyond today + tolerance.
	if d.After(maxDate) {
		return time.Time{}, false
	}
	return d, true
}

// extractDocumentDateWithScore returns the best belegdatum candidate and its
// confidence. found=false when no plausible date exists in the text.
func extractDocumentDateWithScore(ocrText string) (best time.Time, score float64, found bool) {
	if ocrText == "" {
		return time.Time{}, 0, false
	}
	lower := strings.ToLower(ocrText)
	idxMatches := dateCandidateRe.FindAllStringSubmatchIndex(ocrText, -1)
	names := dateCandidateRe.SubexpNames()
	for _, loc := range idxMatches {
		m := make([]string, len(names))
		for g := range names {
			s, e := loc[2*g], loc[2*g+1]
			if s >= 0 {
				m[g] = ocrText[s:e]
			}
		}
		d, ok := parseDateMatch(names, m)
		if !ok {
			continue
		}
		sc := scoreForDatePosition(lower, loc[0])
		// Strictly greater keeps the earliest occurrence on a tie (matches are
		// returned in reading order).
		if !found || sc > score {
			best, score, found = d, sc, true
		}
	}
	return best, score, found
}

// extractDocumentDate returns just the best belegdatum candidate (or nil),
// preserving the original signature for callers that do not need the score.
func extractDocumentDate(ocrText string) *time.Time {
	d, _, found := extractDocumentDateWithScore(ocrText)
	if !found {
		return nil
	}
	return &d
}