FDN-01: repository & projektgerüst
Git-Repository für bestehenden archivdms-Code initialisiert, Branch-/Commit-Konvention (feature/<ticket>-<slug>-Branches, Ticket-Prefix in Commit-Nachricht) etabliert.
This commit is contained in:
@@ -0,0 +1,219 @@
|
||||
// Package matching implements the classification-matching algorithms used
|
||||
// by tags/document_types/correspondents (internal/storage/taxonomy.go) to
|
||||
// auto-assign themselves to a newly ingested document based on its OCR text
|
||||
// plus title. Deliberately dependency-free — no fuzzy-matching Go module is
|
||||
// added (project style: avoid unnecessary deps, see internal/ocr package
|
||||
// comment) — the fuzzy algorithm is a small self-contained normalized
|
||||
// Levenshtein ratio.
|
||||
package matching
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Supported algorithm names, mirrored by the match_algorithm CHECK
|
||||
// constraint on tags/document_types/correspondents.
|
||||
const (
|
||||
AlgorithmNone = "none"
|
||||
AlgorithmAny = "any"
|
||||
AlgorithmAll = "all"
|
||||
AlgorithmExact = "exact"
|
||||
AlgorithmRegex = "regex"
|
||||
AlgorithmFuzzy = "fuzzy"
|
||||
)
|
||||
|
||||
// fuzzyThreshold is the minimum normalized similarity ratio (0..1) for a
|
||||
// fuzzy match to count as a hit.
|
||||
const fuzzyThreshold = 0.85
|
||||
|
||||
// Match reports whether pattern matches somewhere in text, using the given
|
||||
// algorithm and case-sensitivity. Unknown algorithms and "none" always
|
||||
// return false (never auto-assigned). Malformed regex patterns return
|
||||
// false rather than panicking — callers are expected to log this
|
||||
// separately if desired.
|
||||
func Match(algorithm, pattern string, caseSensitive bool, text string) bool {
|
||||
if strings.TrimSpace(pattern) == "" {
|
||||
return false
|
||||
}
|
||||
if !caseSensitive {
|
||||
text = strings.ToLower(text)
|
||||
pattern = strings.ToLower(pattern)
|
||||
}
|
||||
|
||||
switch algorithm {
|
||||
case AlgorithmAny:
|
||||
return matchTerms(pattern, text, false)
|
||||
case AlgorithmAll:
|
||||
return matchTerms(pattern, text, true)
|
||||
case AlgorithmExact:
|
||||
return strings.Contains(text, pattern)
|
||||
case AlgorithmRegex:
|
||||
re, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return re.MatchString(text)
|
||||
case AlgorithmFuzzy:
|
||||
return fuzzyContains(pattern, text)
|
||||
case AlgorithmNone:
|
||||
return false
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// FuzzyThreshold is the minimum FuzzyScore at which the fuzzy Match
|
||||
// algorithm counts as a hit. Exported so suggestion providers (see
|
||||
// internal/storage/metadata_suggestions.go) can pick their own lower floor
|
||||
// relative to the auto-assign threshold.
|
||||
const FuzzyThreshold = fuzzyThreshold
|
||||
|
||||
// FuzzyScore returns the best normalized similarity ratio (0..1) between
|
||||
// pattern and any whitespace-delimited window of text of pattern's own
|
||||
// word-count length, using the same normalized Levenshtein ratio and
|
||||
// word-window scan as the fuzzy Match algorithm. Unlike Match it returns the
|
||||
// raw score instead of a bool threshold decision, so callers can surface
|
||||
// near-miss candidates that scored below FuzzyThreshold. Returns 0 for an
|
||||
// empty pattern. This does NOT change the fuzzy Match algorithm — it only
|
||||
// exposes its underlying score.
|
||||
func FuzzyScore(pattern string, caseSensitive bool, text string) float64 {
|
||||
if strings.TrimSpace(pattern) == "" {
|
||||
return 0
|
||||
}
|
||||
if !caseSensitive {
|
||||
text = strings.ToLower(text)
|
||||
pattern = strings.ToLower(pattern)
|
||||
}
|
||||
patternWords := strings.Fields(pattern)
|
||||
if len(patternWords) == 0 {
|
||||
return 0
|
||||
}
|
||||
textWords := strings.Fields(text)
|
||||
n := len(patternWords)
|
||||
if len(textWords) < n {
|
||||
return levenshteinRatio(pattern, text)
|
||||
}
|
||||
best := 0.0
|
||||
for i := 0; i+n <= len(textWords); i++ {
|
||||
window := strings.Join(textWords[i:i+n], " ")
|
||||
if r := levenshteinRatio(pattern, window); r > best {
|
||||
best = r
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// tokenizePattern splits pattern into terms, honoring "quoted multi-word
|
||||
// terms" as single tokens (e.g. `invoice "Muster GmbH" urgent`).
|
||||
func tokenizePattern(pattern string) []string {
|
||||
var terms []string
|
||||
var cur strings.Builder
|
||||
inQuotes := false
|
||||
flush := func() {
|
||||
if t := strings.TrimSpace(cur.String()); t != "" {
|
||||
terms = append(terms, t)
|
||||
}
|
||||
cur.Reset()
|
||||
}
|
||||
for _, r := range pattern {
|
||||
switch {
|
||||
case r == '"':
|
||||
inQuotes = !inQuotes
|
||||
if !inQuotes {
|
||||
flush()
|
||||
}
|
||||
case r == ' ' && !inQuotes:
|
||||
flush()
|
||||
default:
|
||||
cur.WriteRune(r)
|
||||
}
|
||||
}
|
||||
flush()
|
||||
return terms
|
||||
}
|
||||
|
||||
// matchTerms implements the any/all algorithms: pattern is tokenized into
|
||||
// (possibly quoted, multi-word) terms; requireAll selects "all" vs "any".
|
||||
func matchTerms(pattern, text string, requireAll bool) bool {
|
||||
terms := tokenizePattern(pattern)
|
||||
if len(terms) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, term := range terms {
|
||||
hit := strings.Contains(text, term)
|
||||
if requireAll && !hit {
|
||||
return false
|
||||
}
|
||||
if !requireAll && hit {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return requireAll
|
||||
}
|
||||
|
||||
// fuzzyContains reports whether any whitespace-delimited window of text (of
|
||||
// pattern's own word-count length) is within fuzzyThreshold similarity of
|
||||
// pattern, using a normalized Levenshtein ratio. This is intentionally
|
||||
// simple (word-window scan, not a full substring-alignment fuzzy search) —
|
||||
// adequate for short tag/correspondent names against OCR text.
|
||||
func fuzzyContains(pattern, text string) bool {
|
||||
patternWords := strings.Fields(pattern)
|
||||
if len(patternWords) == 0 {
|
||||
return false
|
||||
}
|
||||
textWords := strings.Fields(text)
|
||||
n := len(patternWords)
|
||||
if len(textWords) < n {
|
||||
return levenshteinRatio(pattern, text) >= fuzzyThreshold
|
||||
}
|
||||
for i := 0; i+n <= len(textWords); i++ {
|
||||
window := strings.Join(textWords[i:i+n], " ")
|
||||
if levenshteinRatio(pattern, window) >= fuzzyThreshold {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// levenshteinRatio returns a normalized similarity ratio in [0,1]: 1 means
|
||||
// identical strings, 0 means completely dissimilar (edit distance equal to
|
||||
// the longer string's length).
|
||||
func levenshteinRatio(a, b string) float64 {
|
||||
maxLen := max(len([]rune(a)), len([]rune(b)))
|
||||
if maxLen == 0 {
|
||||
return 1
|
||||
}
|
||||
dist := levenshteinDistance(a, b)
|
||||
return 1 - float64(dist)/float64(maxLen)
|
||||
}
|
||||
|
||||
// levenshteinDistance computes the classic edit distance between two
|
||||
// strings (rune-aware), using a single-row dynamic-programming table to
|
||||
// keep memory usage O(min(len(a),len(b))).
|
||||
func levenshteinDistance(a, b string) int {
|
||||
ra, rb := []rune(a), []rune(b)
|
||||
if len(ra) < len(rb) {
|
||||
ra, rb = rb, ra
|
||||
}
|
||||
prev := make([]int, len(rb)+1)
|
||||
curr := make([]int, len(rb)+1)
|
||||
for j := range prev {
|
||||
prev[j] = j
|
||||
}
|
||||
for i := 1; i <= len(ra); i++ {
|
||||
curr[0] = i
|
||||
for j := 1; j <= len(rb); j++ {
|
||||
cost := 1
|
||||
if ra[i-1] == rb[j-1] {
|
||||
cost = 0
|
||||
}
|
||||
del := prev[j] + 1
|
||||
ins := curr[j-1] + 1
|
||||
sub := prev[j-1] + cost
|
||||
curr[j] = min(del, min(ins, sub))
|
||||
}
|
||||
prev, curr = curr, prev
|
||||
}
|
||||
return prev[len(rb)]
|
||||
}
|
||||
Reference in New Issue
Block a user