FDN-01: repository & projektgerüst
Git-Repository für bestehenden archivdms-Code initialisiert, Branch-/Commit-Konvention (feature/<ticket>-<slug>-Branches, Ticket-Prefix in Commit-Nachricht) etabliert.
This commit is contained in:
@@ -0,0 +1,216 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// extractDocumentDate scans OCR text for the most plausible document/invoice
|
||||
// date and returns it, or nil if none is found. It is a deliberately simple,
|
||||
// rule-based (no NLP, no external library, CGO-free) heuristic in the same
|
||||
// spirit as titleFromOCRText.
|
||||
//
|
||||
// Recognised formats:
|
||||
// - DD.MM.YYYY (German, e.g. 31.12.2024)
|
||||
// - DD.MM.YY (German short year, e.g. 31.12.24 -> 2024)
|
||||
// - DD/MM/YYYY (slash variant, incl. DD/MM/YY)
|
||||
// - YYYY-MM-DD (ISO 8601, e.g. 2024-12-31)
|
||||
// - "15. März 2026" / "15. Mär. 2026" (spelled-out German month names)
|
||||
//
|
||||
// Plausibility: month 1-12, day 1-31 with an explicit calendar check (time.Date
|
||||
// would normalise an impossible day, so we reject e.g. "31.02." rather than
|
||||
// silently shifting it to March), year not before dateMinYear, and never a date
|
||||
// in the future beyond today + dateFutureToleranceDays (a small tolerance for
|
||||
// timezone/clock skew). These filters drop copyright years, footer years and
|
||||
// stray digit runs.
|
||||
//
|
||||
// Scoring (see scoreForDatePosition): each candidate gets a confidence based on
|
||||
// signal words in a small text window around it ("Rechnungsdatum", "vom", ...).
|
||||
// The candidate with the highest score wins; on a tie the earliest occurrence in
|
||||
// the text wins (document head = usually the issue date). Callers that only need
|
||||
// the date use this function; callers that also need the confidence use
|
||||
// extractDocumentDateWithScore.
|
||||
const (
|
||||
dateMinYear = 1990
|
||||
dateFutureToleranceDays = 2
|
||||
dateWindowRadius = 40
|
||||
dateScoreNoContext = 0.4
|
||||
)
|
||||
|
||||
// dateKeyword pairs a lowercase signal word with the confidence a date near it
|
||||
// receives. Ordered by descending score: scoreForDatePosition returns the score
|
||||
// of the first (=highest) keyword found in the window. Kept short and flat on
|
||||
// purpose — no weighting engine, GoBD-traceable.
|
||||
type dateKeyword struct {
|
||||
word string
|
||||
score float64
|
||||
}
|
||||
|
||||
var dateKeywords = []dateKeyword{
|
||||
{"rechnungsdatum", 0.9},
|
||||
{"belegdatum", 0.9},
|
||||
{"ausstellungsdatum", 0.9},
|
||||
{"rechnung vom", 0.9},
|
||||
{"beleg vom", 0.9},
|
||||
{"datum", 0.75},
|
||||
{"vom", 0.55},
|
||||
}
|
||||
|
||||
// dateGermanMonths maps lowercased German month names and common abbreviations
|
||||
// (with the trailing dot already stripped) to their month number.
|
||||
var dateGermanMonths = map[string]int{
|
||||
"januar": 1, "jan": 1,
|
||||
"februar": 2, "feb": 2,
|
||||
"märz": 3, "maerz": 3, "mär": 3, "mrz": 3,
|
||||
"april": 4, "apr": 4,
|
||||
"mai": 5,
|
||||
"juni": 6, "jun": 6,
|
||||
"juli": 7, "jul": 7,
|
||||
"august": 8, "aug": 8,
|
||||
"september": 9, "sep": 9, "sept": 9,
|
||||
"oktober": 10, "okt": 10,
|
||||
"november": 11, "nov": 11,
|
||||
"dezember": 12, "dez": 12,
|
||||
}
|
||||
|
||||
// dateCandidateRe matches every supported format in a single alternation. Named
|
||||
// groups keep the branch handling readable. Word boundaries avoid gluing onto
|
||||
// surrounding digits (e.g. a phone number). Case-insensitive for month names.
|
||||
var dateCandidateRe = regexp.MustCompile(
|
||||
`(?i)(?:\b(?P<gd>\d{1,2})\.(?P<gm>\d{1,2})\.(?P<gy>\d{4}|\d{2})\b)` +
|
||||
`|(?:\b(?P<sd>\d{1,2})/(?P<sm>\d{1,2})/(?P<sy>\d{4}|\d{2})\b)` +
|
||||
`|(?:\b(?P<iy>\d{4})-(?P<im>\d{1,2})-(?P<id>\d{1,2})\b)` +
|
||||
`|(?:\b(?P<td>\d{1,2})\.?\s+(?P<tmon>[A-Za-zäöüÄÖÜ]+)\.?\s+(?P<ty>\d{4})\b)`,
|
||||
)
|
||||
|
||||
// sameDate reports whether two optional dates refer to the same calendar day
|
||||
// (or are both nil). Used by reprocess to skip a no-op document_date update.
|
||||
func sameDate(a, b *time.Time) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == nil && b == nil
|
||||
}
|
||||
ay, am, ad := a.Date()
|
||||
by, bm, bd := b.Date()
|
||||
return ay == by && am == bm && ad == bd
|
||||
}
|
||||
|
||||
// scoreForDatePosition returns the confidence for a date match found at byte
|
||||
// offset start in text, based on signal words within dateWindowRadius chars.
|
||||
func scoreForDatePosition(lowerText string, start int) float64 {
|
||||
lo := start - dateWindowRadius
|
||||
if lo < 0 {
|
||||
lo = 0
|
||||
}
|
||||
hi := start + dateWindowRadius
|
||||
if hi > len(lowerText) {
|
||||
hi = len(lowerText)
|
||||
}
|
||||
window := lowerText[lo:hi]
|
||||
for _, kw := range dateKeywords {
|
||||
if strings.Contains(window, kw.word) {
|
||||
return kw.score
|
||||
}
|
||||
}
|
||||
return dateScoreNoContext
|
||||
}
|
||||
|
||||
// parseDateMatch turns one regex submatch into a validated calendar date, or
|
||||
// returns ok=false if the match is implausible.
|
||||
func parseDateMatch(names, m []string) (time.Time, bool) {
|
||||
now := time.Now()
|
||||
maxDate := now.AddDate(0, 0, dateFutureToleranceDays)
|
||||
var day, month, year int
|
||||
var monthName string
|
||||
for i, name := range names {
|
||||
if m[i] == "" {
|
||||
continue
|
||||
}
|
||||
switch name {
|
||||
case "gd", "sd", "id", "td":
|
||||
day, _ = strconv.Atoi(m[i])
|
||||
case "gm", "sm", "im":
|
||||
month, _ = strconv.Atoi(m[i])
|
||||
case "gy", "sy":
|
||||
y, _ := strconv.Atoi(m[i])
|
||||
if len(m[i]) == 2 {
|
||||
// Two-digit year: interpret as 2000-2099. Anything above the
|
||||
// future tolerance is rejected below.
|
||||
y += 2000
|
||||
}
|
||||
year = y
|
||||
case "iy", "ty":
|
||||
year, _ = strconv.Atoi(m[i])
|
||||
case "tmon":
|
||||
monthName = m[i]
|
||||
}
|
||||
}
|
||||
if monthName != "" {
|
||||
mn, ok := dateGermanMonths[strings.ToLower(monthName)]
|
||||
if !ok {
|
||||
return time.Time{}, false
|
||||
}
|
||||
month = mn
|
||||
}
|
||||
if year < dateMinYear {
|
||||
return time.Time{}, false
|
||||
}
|
||||
if month < 1 || month > 12 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
if day < 1 || day > 31 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
d := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC)
|
||||
// Reject normalised-away impossible days (e.g. 31.02. -> 03.03.).
|
||||
if d.Day() != day || int(d.Month()) != month || d.Year() != year {
|
||||
return time.Time{}, false
|
||||
}
|
||||
// No future dates beyond today + tolerance.
|
||||
if d.After(maxDate) {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
|
||||
// extractDocumentDateWithScore returns the best belegdatum candidate and its
|
||||
// confidence. found=false when no plausible date exists in the text.
|
||||
func extractDocumentDateWithScore(ocrText string) (best time.Time, score float64, found bool) {
|
||||
if ocrText == "" {
|
||||
return time.Time{}, 0, false
|
||||
}
|
||||
lower := strings.ToLower(ocrText)
|
||||
idxMatches := dateCandidateRe.FindAllStringSubmatchIndex(ocrText, -1)
|
||||
names := dateCandidateRe.SubexpNames()
|
||||
for _, loc := range idxMatches {
|
||||
m := make([]string, len(names))
|
||||
for g := range names {
|
||||
s, e := loc[2*g], loc[2*g+1]
|
||||
if s >= 0 {
|
||||
m[g] = ocrText[s:e]
|
||||
}
|
||||
}
|
||||
d, ok := parseDateMatch(names, m)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
sc := scoreForDatePosition(lower, loc[0])
|
||||
// Strictly greater keeps the earliest occurrence on a tie (matches are
|
||||
// returned in reading order).
|
||||
if !found || sc > score {
|
||||
best, score, found = d, sc, true
|
||||
}
|
||||
}
|
||||
return best, score, found
|
||||
}
|
||||
|
||||
// extractDocumentDate returns just the best belegdatum candidate (or nil),
|
||||
// preserving the original signature for callers that do not need the score.
|
||||
func extractDocumentDate(ocrText string) *time.Time {
|
||||
d, _, found := extractDocumentDateWithScore(ocrText)
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
return &d
|
||||
}
|
||||
Reference in New Issue
Block a user