SRC-08: facetten- & filter-api
CI / Core-Modul (Build/Vet/Test) (push) Canceled after 0s
CI / DMS Security-Test-Gate (AC5, Merge-Gate) (push) Canceled after 0s
CI / DMS-Modul (Lint/Build/Test/Coverage/Artefakt) (push) Canceled after 0s
CI / DMS-Modul Supply-Chain-Scan (govulncheck) (push) Canceled after 0s
CI / Web (dms-browser) (push) Canceled after 0s
CI / Web (pdf-viewer) (push) Canceled after 0s
CI / Web (shl) (push) Canceled after 0s

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
patrickandClaude Sonnet 5 committed 2026-09-20 01:01:20 +02:00
1 parent f45cf29184
commit 2a49782510
11 files changed
+1013 -25

No files matched your search

@@ -7,6 +7,10 @@
-- in internal/searchindex.BuildSearchQuery statt hier im Schema:
-- title = 10, tags = 5, body = 1
--
-- attrs (SRC-08): JSON mit tags[] (Tag-Namen), f{} (numerische/Datums-
-- Werte benutzerdefinierter Felder, Datum als Epoche) und fs{} (Rohwert je
-- Feld als String) fuer Facetten und Filter.
--
-- Deutsche Sprachbehandlung (Akzeptanzkriterium 3):
-- morphology='libstemmer_de' -- Snowball-Stemmer fuer Deutsch
-- charset_table erweitert um Umlaute/ß als Buchstaben (sonst als
@@ -25,7 +29,8 @@ CREATE TABLE IF NOT EXISTS dms_documents (
document_type string attribute,
created_by string attribute,
created_at timestamp,
updated_at timestamp
updated_at timestamp,
attrs json
)
morphology='libstemmer_de'
charset_table='non_cjk, U+00DF, U+00E4, U+00F6, U+00FC, U+00C4, U+00D6, U+00DC'
+351
View File
@@ -0,0 +1,351 @@
package searchapi
import (
"context"
"fmt"
"net/url"
"os"
"sort"
"strings"
"testing"
"time"
"gitea.perlbach24.de/scripte/nexarch/dms/internal/documenttype"
"gitea.perlbach24.de/scripte/nexarch/dms/internal/metadatafield"
"gitea.perlbach24.de/scripte/nexarch/dms/internal/searchindex"
"gitea.perlbach24.de/scripte/nexarch/dms/internal/searchsync"
"gitea.perlbach24.de/scripte/nexarch/dms/internal/tag"
"gitea.perlbach24.de/scripte/nexarch/dms/internal/testenv"
)
func facetMap(vals []facetValueJSON) map[string]int {
m := map[string]int{}
for _, v := range vals {
m[v.Value] = v.Count
}
return m
}
func docIDs(out responseJSON) []string {
var ids []string
for _, h := range out.Hits {
ids = append(ids, h.DocumentID)
}
sort.Strings(ids)
return ids
}
func attrsOf(tags []string, typed map[string]float64, strs map[string]string) searchindex.Attrs {
return searchindex.Attrs{Tags: tags, Typed: typed, Strings: strs}
}
func TestFacets_MultipleFiltersCombine(t *testing.T) {
srv, db := setup(t, denyList{})
tok := token(t)
now := time.Now().Unix()
a := index(t, db, searchindex.Doc{Title: tok + " a", CreatedBy: "u1", DocumentType: "Rechnung", UpdatedAtSec: now,
Attrs: attrsOf([]string{"steuer", "2026"}, map[string]float64{"betrag": 100}, map[string]string{"region": "nord"})})
index(t, db, searchindex.Doc{Title: tok + " b", CreatedBy: "u1", DocumentType: "Rechnung", UpdatedAtSec: now,
Attrs: attrsOf([]string{"steuer"}, map[string]float64{"betrag": 900}, map[string]string{"region": "nord"})})
index(t, db, searchindex.Doc{Title: tok + " c", CreatedBy: "u1", DocumentType: "Vertrag", UpdatedAtSec: now,
Attrs: attrsOf([]string{"steuer", "2026"}, map[string]float64{"betrag": 100}, map[string]string{"region": "sued"})})
base := srv.URL + "/search?caller=u1&q=" + tok
cases := []struct {
name string
query string
want int
}{
{"nur typ", "&document_type=Rechnung", 2},
{"typ+tag", "&document_type=Rechnung&tag=2026", 1},
{"typ+tag+feldbereich", "&document_type=Rechnung&tag=2026&field.betrag.min=50&field.betrag.max=150", 1},
{"zwei tags", "&tag=steuer&tag=2026", 2},
{"feld-gleichheit+typ", "&field.region=nord&document_type=Rechnung", 2},
{"typ-oder", "&document_type=Rechnung&document_type=Vertrag", 3},
{"widerspruch", "&document_type=Vertrag&field.region=nord", 0},
}
for _, c := range cases {
status, out := get(t, base+c.query)
if status != 200 || out.Total != c.want || len(out.Hits) != c.want {
t.Errorf("%s: status=%d total=%d hits=%d, want %d", c.name, status, out.Total, len(out.Hits), c.want)
}
}
_, out := get(t, base+"&document_type=Rechnung&tag=2026&field.betrag.max=150")
if ids := docIDs(out); len(ids) != 1 || ids[0] != a.DocumentID {
t.Errorf("falsches dokument: %v", ids)
}
}
func TestFacets_CountsPerValueAndUnderActiveFilters(t *testing.T) {
srv, db := setup(t, denyList{})
tok := token(t)
now := time.Now().Unix()
for i, spec := range []struct {
typ string
tags []string
reg string
}{
{"Rechnung", []string{"a", "b"}, "nord"}, {"Rechnung", []string{"a"}, "sued"}, {"Rechnung", nil, "nord"},
{"Vertrag", []string{"b"}, "nord"}, {"", []string{"a"}, ""},
} {
index(t, db, searchindex.Doc{Title: fmt.Sprintf("%s %d", tok, i), CreatedBy: "u1", DocumentType: spec.typ, UpdatedAtSec: now,
Attrs: attrsOf(spec.tags, nil, map[string]string{"region": spec.reg})})
}
base := srv.URL + "/search?caller=u1&q=" + tok + "&facets=document_type,tag,created_by,field.region"
_, out := get(t, base)
if out.Total != 5 {
t.Fatalf("total = %d", out.Total)
}
if got := facetMap(out.Facets["document_type"]); got["Rechnung"] != 3 || got["Vertrag"] != 1 || len(got) != 2 {
t.Errorf("document_type = %v", got)
}
if got := facetMap(out.Facets["tag"]); got["a"] != 3 || got["b"] != 2 || len(got) != 2 {
t.Errorf("tag = %v", got)
}
if got := facetMap(out.Facets["created_by"]); got["u1"] != 5 || len(got) != 1 {
t.Errorf("created_by = %v", got)
}
if got := facetMap(out.Facets["field.region"]); got["nord"] != 3 || got["sued"] != 1 || len(got) != 2 {
t.Errorf("field.region = %v", got)
}
_, out = get(t, base+"&tag=a")
if got := facetMap(out.Facets["document_type"]); got["Rechnung"] != 2 || len(got) != 1 {
t.Errorf("document_type unter tag=a = %v", got)
}
if got := facetMap(out.Facets["tag"]); got["a"] != 3 || got["b"] != 1 {
t.Errorf("tag unter tag=a = %v", got)
}
if out.Facets["tag"][0].Value != "a" {
t.Errorf("facetten nicht nach anzahl absteigend: %v", out.Facets["tag"])
}
}
func TestFilters_DateAndValueRanges(t *testing.T) {
srv, db := setup(t, denyList{})
tok := token(t)
day := func(s string) int64 {
d, _ := time.Parse("2006-01-02", s)
return d.Unix() + 3600
}
for i, spec := range []struct {
updated string
betrag float64
frist string
}{
{"2026-01-10", 10, "2026-03-01"}, {"2026-02-10", 20, "2026-04-01"}, {"2026-03-10", 30, "2026-05-01"},
} {
frist, _ := time.Parse("2006-01-02", spec.frist)
index(t, db, searchindex.Doc{Title: fmt.Sprintf("%s %d", tok, i), CreatedBy: "u1", UpdatedAtSec: day(spec.updated),
Attrs: attrsOf(nil, map[string]float64{"betrag": spec.betrag, "frist": float64(frist.Unix())}, nil)})
}
base := srv.URL + "/search?caller=u1&q=" + tok
for _, c := range []struct {
query string
want int
}{
{"&updated_from=2026-02-01", 2},
{"&updated_to=2026-02-10", 2},
{"&updated_from=2026-02-10&updated_to=2026-02-10", 1},
{"&updated_from=2026-01-15&updated_to=2026-03-01", 1},
{"&created_from=2026-01-01T00:00:00Z", 3},
{"&field.betrag.min=15&field.betrag.max=25", 1},
{"&field.betrag.min=20", 2},
{"&field.frist.min=2026-04-01&field.frist.max=2026-04-30", 1},
{"&field.frist.max=2026-03-31", 1},
} {
if status, out := get(t, base+c.query); status != 200 || out.Total != c.want {
t.Errorf("%s: status=%d total=%d, want %d", c.query, status, out.Total, c.want)
}
}
}
func TestFilters_InvalidParametersRejected(t *testing.T) {
srv, _ := setup(t, denyList{})
base := srv.URL + "/search?caller=u1&q=x"
for _, q := range []string{
"&updated_from=gestern", "&created_from=2026-03-01&created_to=2026-01-01", "&field.Betrag=x", "&field.a.b.min=1",
"&field.betrag.min=abc", "&facets=passwort", "&facets=field.a;drop", "&field.x%3Bdrop=1", "&field.betrag.avg=1",
} {
if status, _ := get(t, base+q); status != 400 {
t.Errorf("%s: status = %d, want 400", q, status)
}
}
}
// Pruefung 2: leeres Ergebnis bleibt stabil (Struktur, Facetten leer, kein Fehler).
func TestFilters_EmptyResultIsStable(t *testing.T) {
srv, db := setup(t, denyList{})
tok := token(t)
index(t, db, searchindex.Doc{Title: tok, CreatedBy: "u1", DocumentType: "Rechnung"})
status, out := get(t, srv.URL+"/search?caller=u1&q="+tok+"&document_type=Nichts&tag=x&field.a.min=1&facets=document_type,tag,field.a")
if status != 200 || out.Total != 0 || out.Hits == nil || len(out.Hits) != 0 {
t.Fatalf("status=%d total=%d hits=%v", status, out.Total, out.Hits)
}
for _, name := range []string{"document_type", "tag", "field.a"} {
vals, ok := out.Facets[name]
if !ok || len(vals) != 0 {
t.Errorf("facette %s = %v (vorhanden=%v), want leer", name, vals, ok)
}
}
}
// Pruefung 3: Antwortzeit mit fuenf aktiven Filtern.
func TestFilters_ResponseTimeWithFiveFilters(t *testing.T) {
srv, db := setup(t, denyList{})
tok := token(t)
now := time.Now().Unix()
for i := 0; i < 200; i++ {
index(t, db, searchindex.Doc{Title: fmt.Sprintf("%s %d", tok, i), CreatedBy: "u1", DocumentType: "Rechnung", UpdatedAtSec: now,
Attrs: attrsOf([]string{"steuer", "2026"}, map[string]float64{"betrag": float64(i)}, map[string]string{"region": "nord"})})
}
q := url.Values{"caller": {"u1"}, "q": {tok}, "document_type": {"Rechnung"}, "tag": {"steuer", "2026"},
"updated_from": {"2020-01-01"}, "field.betrag.min": {"10"}, "field.region": {"nord"}, "facets": {"document_type,tag,field.region"}}
var durations []time.Duration
for i := 0; i < 30; i++ {
start := time.Now()
status, out := get(t, srv.URL+"/search?"+q.Encode())
durations = append(durations, time.Since(start))
if status != 200 || out.Total != 190 {
t.Fatalf("status=%d total=%d", status, out.Total)
}
}
sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] })
p95 := durations[len(durations)*95/100]
t.Logf("5 filter + 3 facetten: median=%v p95=%v max=%v", durations[len(durations)/2], p95, durations[len(durations)-1])
if p95 > responseTimeLimit {
t.Errorf("p95 = %v ueber %v", p95, responseTimeLimit)
}
}
// Pruefung 1: Facetten- und Trefferzahlen gegen SQL auf der Quelldatenbank abgeglichen.
func TestFacets_CountsMatchDatabaseQuery(t *testing.T) {
if os.Getenv("TEST_TENANT_DSN") == "" {
t.Skip("TEST_TENANT_DSN nicht gesetzt")
}
srv, idx := setup(t, denyList{})
e := testenv.New(t)
ctx := context.Background()
syncer := &searchsync.Syncer{Pool: e.Pool, Index: idx}
tok := token(t)
types := map[string]string{}
for _, n := range []string{"Rechnung", "Vertrag"} {
dt, err := documenttype.Create(ctx, e.Pool, n, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
types[n] = dt.ID
}
tags := map[string]string{}
for _, n := range []string{"steuer", "2026", "intern"} {
tg, err := tag.Create(ctx, e.Pool, n, "", nil)
if err != nil {
t.Fatal(err)
}
tags[n] = tg.ID
}
fields := map[string]string{}
for key, typ := range map[string]string{"betrag": "number", "region": "select", "frist": "date"} {
f, err := metadatafield.Create(ctx, e.Pool, key, key, typ, false, nil, nil)
if err != nil {
t.Fatalf("feld %s: %v", key, err)
}
fields[key] = f.ID
}
typeNames := []string{"Rechnung", "Vertrag"}
tagNames := []string{"steuer", "2026", "intern"}
regions := []string{"nord", "sued", "west"}
const n = 36
for i := 0; i < n; i++ {
id, _ := e.UploadDoc(t, fmt.Sprintf("%s-%02d.txt", tok, i), "text/plain", []byte(fmt.Sprintf("inhalt %s %d", tok, i)))
t.Cleanup(func() { _ = searchindex.Delete(ctx, idx, searchsync.DocIndexID(id)) })
if _, err := documenttype.ChangeDocumentType(ctx, e.Pool, id, types[typeNames[i%2]], false); err != nil {
t.Fatal(err)
}
for j, tn := range tagNames {
if (i>>j)&1 == 1 || (i%5 == 0 && j == 0) {
if err := tag.Assign(ctx, e.Pool, id, tags[tn]); err != nil {
t.Fatal(err)
}
}
}
for key, val := range map[string]string{
"betrag": fmt.Sprintf("%d", (i*37)%500), "region": regions[i%3],
"frist": time.Date(2026, time.Month(1+i%6), 1+i%27, 0, 0, 0, 0, time.UTC).Format("2006-01-02"),
} {
if err := metadatafield.SetValue(ctx, e.Pool, id, fields[key], val); err != nil {
t.Fatalf("wert %s=%s: %v", key, val, err)
}
}
if err := syncer.SyncDocument(ctx, id); err != nil {
t.Fatalf("sync: %v", err)
}
}
count := func(sqlText string, args ...any) int {
t.Helper()
var c int
if err := e.Pool.QueryRow(ctx, sqlText, args...).Scan(&c); err != nil {
t.Fatalf("sql: %v", err)
}
return c
}
facetsQ := "&facets=document_type,tag,field.region"
filterQ := "&document_type=Rechnung&tag=steuer&field.betrag.min=100&field.betrag.max=400&field.frist.min=2026-02-01&field.frist.max=2026-05-31"
const filterSQL = `
FROM documents d
JOIN document_types dt ON dt.id = d.document_type_id
WHERE d.title LIKE $1 AND dt.name = 'Rechnung'
AND EXISTS (SELECT 1 FROM document_tags x JOIN tags t ON t.id = x.tag_id WHERE x.document_id = d.id AND t.name = 'steuer')
AND EXISTS (SELECT 1 FROM document_metadata_values v JOIN metadata_fields f ON f.id = v.field_id
WHERE v.document_id = d.id AND f.field_key = 'betrag' AND v.value::numeric BETWEEN 100 AND 400)
AND EXISTS (SELECT 1 FROM document_metadata_values v JOIN metadata_fields f ON f.id = v.field_id
WHERE v.document_id = d.id AND f.field_key = 'frist' AND v.value::date BETWEEN '2026-02-01' AND '2026-05-31')`
like := tok + "%"
base := srv.URL + "/search?caller=" + e.UserID + "&limit=100&q=" + tok
_, all := get(t, base+facetsQ)
if want := count(`SELECT count(*) FROM documents WHERE title LIKE $1`, like); all.Total != want || want != n {
t.Fatalf("gesamt: index=%d db=%d, erwartet %d", all.Total, want, n)
}
for _, name := range typeNames {
want := count(`SELECT count(*) FROM documents d JOIN document_types dt ON dt.id = d.document_type_id WHERE d.title LIKE $1 AND dt.name = $2`, like, name)
if got := facetMap(all.Facets["document_type"])[name]; got != want {
t.Errorf("facette typ %s: index=%d db=%d", name, got, want)
}
}
for _, name := range tagNames {
want := count(`SELECT count(DISTINCT d.id) FROM documents d JOIN document_tags x ON x.document_id = d.id JOIN tags t ON t.id = x.tag_id WHERE d.title LIKE $1 AND t.name = $2`, like, name)
if got := facetMap(all.Facets["tag"])[name]; got != want {
t.Errorf("facette tag %s: index=%d db=%d", name, got, want)
}
}
for _, r := range regions {
want := count(`SELECT count(*) FROM documents d JOIN document_metadata_values v ON v.document_id = d.id JOIN metadata_fields f ON f.id = v.field_id WHERE d.title LIKE $1 AND f.field_key = 'region' AND v.value = $2`, like, r)
if got := facetMap(all.Facets["field.region"])[r]; got != want {
t.Errorf("facette region %s: index=%d db=%d", r, got, want)
}
}
_, filtered := get(t, base+filterQ+facetsQ)
wantTotal := count(`SELECT count(*) `+filterSQL, like)
if wantTotal == 0 {
t.Fatal("testdaten ergeben leere filtermenge, pruefung sinnlos")
}
if filtered.Total != wantTotal || len(filtered.Hits) != wantTotal {
t.Errorf("gefilterte treffer: index=%d/%d db=%d", filtered.Total, len(filtered.Hits), wantTotal)
}
for _, r := range regions {
want := count(`SELECT count(*) `+filterSQL+` AND EXISTS (SELECT 1 FROM document_metadata_values v JOIN metadata_fields f ON f.id = v.field_id WHERE v.document_id = d.id AND f.field_key = 'region' AND v.value = $2)`, like, r)
if got := facetMap(filtered.Facets["field.region"])[r]; got != want {
t.Errorf("gefilterte facette region %s: index=%d db=%d", r, got, want)
}
}
if got := facetMap(filtered.Facets["document_type"]); len(got) != 1 || got["Rechnung"] != wantTotal {
t.Errorf("gefilterte facette typ = %v, want Rechnung=%d", got, wantTotal)
}
if strings.TrimSpace(tok) == "" {
t.Fatal("token leer")
}
}
+168
View File
@@ -0,0 +1,168 @@
package searchapi
import (
"fmt"
"net/url"
"sort"
"strconv"
"strings"
"time"
"gitea.perlbach24.de/scripte/nexarch/dms/internal/searchindex"
)
const fieldPrefix = "field."
func invalid(format string, a ...any) error {
return fmt.Errorf("%w: %s", searchindex.ErrInvalidRequest, fmt.Sprintf(format, a...))
}
func facetName(s searchindex.FacetSpec) string {
switch s.Kind {
case searchindex.FacetDocumentType:
return "document_type"
case searchindex.FacetTag:
return "tag"
case searchindex.FacetCreatedBy:
return "created_by"
default:
return fieldPrefix + s.Key
}
}
// parseTime akzeptiert YYYY-MM-DD oder RFC3339; endOfDay macht ein reines
// Datum zur letzten Sekunde des Tages (obere Bereichsgrenze).
func parseTime(v string, endOfDay bool) (int64, error) {
if d, err := time.Parse("2006-01-02", v); err == nil {
if endOfDay {
d = d.Add(24*time.Hour - time.Second)
}
return d.Unix(), nil
}
t, err := time.Parse(time.RFC3339, v)
if err != nil {
return 0, invalid("zeitwert %q ungueltig (YYYY-MM-DD oder RFC3339)", v)
}
return t.Unix(), nil
}
func parseBound(v string, endOfDay bool) (float64, error) {
if n, err := strconv.ParseFloat(v, 64); err == nil {
return n, nil
}
t, err := parseTime(v, endOfDay)
if err != nil {
return 0, invalid("bereichswert %q ist weder zahl noch datum", v)
}
return float64(t), nil
}
func parseFacetParams(q url.Values) (searchindex.FacetFilters, []searchindex.FacetSpec, error) {
var ff searchindex.FacetFilters
var specs []searchindex.FacetSpec
ff.DocumentTypes = nonEmpty(q["document_type"])
ff.Tags = nonEmpty(q["tag"])
for _, r := range []struct {
field searchindex.RangeField
from, to string
}{
{searchindex.RangeCreatedAt, "created_from", "created_to"},
{searchindex.RangeUpdatedAt, "updated_from", "updated_to"},
} {
rg := searchindex.Range{Field: r.field}
if v := q.Get(r.from); v != "" {
t, err := parseTime(v, false)
if err != nil {
return ff, nil, err
}
rg.Min = &t
}
if v := q.Get(r.to); v != "" {
t, err := parseTime(v, true)
if err != nil {
return ff, nil, err
}
rg.Max = &t
}
if rg.Min != nil && rg.Max != nil && *rg.Min > *rg.Max {
return ff, nil, invalid("%s liegt nach %s", r.from, r.to)
}
if rg.Min != nil || rg.Max != nil {
ff.Ranges = append(ff.Ranges, rg)
}
}
custom := map[string]*searchindex.CustomFilter{}
var keys []string
for name, vals := range q {
if !strings.HasPrefix(name, fieldPrefix) || len(vals) == 0 || vals[0] == "" {
continue
}
key, part := strings.TrimPrefix(name, fieldPrefix), ""
if i := strings.LastIndex(key, "."); i >= 0 {
key, part = key[:i], key[i+1:]
if part != "min" && part != "max" {
return ff, nil, invalid("parameter %q unbekannt", name)
}
}
if !searchindex.ValidCustomKey(key) {
return ff, nil, invalid("feldschluessel %q nicht erlaubt", key)
}
c := custom[key]
if c == nil {
c = &searchindex.CustomFilter{Key: key}
custom[key] = c
keys = append(keys, key)
}
switch part {
case "":
v := vals[0]
c.Eq = &v
case "min":
n, err := parseBound(vals[0], false)
if err != nil {
return ff, nil, err
}
c.Min = &n
case "max":
n, err := parseBound(vals[0], true)
if err != nil {
return ff, nil, err
}
c.Max = &n
}
}
sort.Strings(keys)
for _, k := range keys {
ff.Custom = append(ff.Custom, *custom[k])
}
if v := strings.TrimSpace(q.Get("facets")); v != "" {
for _, name := range strings.Split(v, ",") {
switch name = strings.TrimSpace(name); {
case name == "document_type":
specs = append(specs, searchindex.FacetSpec{Kind: searchindex.FacetDocumentType})
case name == "tag":
specs = append(specs, searchindex.FacetSpec{Kind: searchindex.FacetTag})
case name == "created_by":
specs = append(specs, searchindex.FacetSpec{Kind: searchindex.FacetCreatedBy})
case strings.HasPrefix(name, fieldPrefix) && searchindex.ValidCustomKey(strings.TrimPrefix(name, fieldPrefix)):
specs = append(specs, searchindex.FacetSpec{Kind: searchindex.FacetCustom, Key: strings.TrimPrefix(name, fieldPrefix)})
default:
return ff, nil, invalid("facette %q nicht erlaubt", name)
}
}
}
return ff, specs, nil
}
func nonEmpty(in []string) []string {
var out []string
for _, v := range in {
if v != "" {
out = append(out, v)
}
}
return out
}
+36 -4
View File
@@ -48,6 +48,13 @@ type responseJSON struct {
Offset int `json:"offset"`
TookMS int64 `json:"took_ms"`
Hits []hitJSON `json:"hits"`
// Facets nur bei gesetztem Parameter "facets"; Schluessel wie dort angegeben.
Facets map[string][]facetValueJSON `json:"facets,omitempty"`
}
type facetValueJSON struct {
Value string `json:"value"`
Count int `json:"count"`
}
func writeError(w http.ResponseWriter, status int, message string) {
@@ -105,7 +112,7 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
return
}
filters := []searchindex.Filter{{Field: searchindex.FilterCreatedBy, Value: caller}}
for _, name := range []string{"document_type", "folder_path"} {
for _, name := range []string{"folder_path", "created_by"} {
if v := q.Get(name); v != "" {
field, err := searchindex.ParseFilterField(name)
if err != nil {
@@ -115,10 +122,16 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
filters = append(filters, searchindex.Filter{Field: field, Value: v})
}
}
facetFilters, specs, err := parseFacetParams(q)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
searchReq := searchindex.SearchRequest{
Term: term, Filters: filters, Sort: sortVal, Limit: limit, Offset: offset, Facet: facetFilters,
}
page, err := searchindex.SearchPage(r.Context(), s.index, searchindex.SearchRequest{
Term: term, Filters: filters, Sort: sortVal, Limit: limit, Offset: offset,
})
page, err := searchindex.SearchPage(r.Context(), s.index, searchReq)
if err != nil {
if errors.Is(err, searchindex.ErrInvalidRequest) {
writeError(w, http.StatusBadRequest, err.Error())
@@ -146,6 +159,25 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
Score: h.Score, UpdatedAt: h.UpdatedAt, SnippetHTML: renderSnippet(h.Snippet),
})
}
if len(specs) > 0 {
results, err := searchindex.Facets(r.Context(), s.index, searchReq, specs)
if err != nil {
if errors.Is(err, searchindex.ErrInvalidRequest) {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeError(w, http.StatusInternalServerError, err.Error())
return
}
resp.Facets = map[string][]facetValueJSON{}
for _, fr := range results {
vals := make([]facetValueJSON, 0, len(fr.Values))
for _, v := range fr.Values {
vals = append(vals, facetValueJSON{Value: v.Value, Count: v.Count})
}
resp.Facets[facetName(fr.Spec)] = vals
}
}
resp.TookMS = time.Since(start).Milliseconds()
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(resp)
+25
View File
@@ -0,0 +1,25 @@
package searchindex
import (
"fmt"
"regexp"
)
// Feldschluessel benutzerdefinierter Felder duerfen als JSON-Pfad in die
// Abfrage; deshalb nur dieses enge Alphabet. Diese Datei ist die einzige
// Stelle, an der ein Nutzerwert Teil des SQL-Textes wird.
var customKeyPattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,62}$`)
func ValidCustomKey(key string) bool { return customKeyPattern.MatchString(key) }
const (
pathTyped = "attrs.f."
pathString = "attrs.fs."
)
func customPath(prefix, key string) (string, error) {
if !ValidCustomKey(key) {
return "", fmt.Errorf("%w: feldschluessel %q nicht erlaubt", ErrInvalidRequest, key)
}
return prefix + key, nil
}
@@ -0,0 +1,67 @@
package searchindex
import (
"errors"
"strings"
"testing"
)
func TestCustomPath_RejectsAnythingButPlainKeys(t *testing.T) {
for _, key := range []string{"", "Betrag", "1abc", "a b", "a.b", "a;drop", "a'b", "a)or(1=1", "a-b", strings.Repeat("a", 64), "ä"} {
if _, err := customPath(pathString, key); !errors.Is(err, ErrInvalidRequest) {
t.Errorf("key %q: err = %v, want ErrInvalidRequest", key, err)
}
}
got, err := customPath(pathTyped, "betrag_netto")
if err != nil || got != "attrs.f.betrag_netto" {
t.Errorf("got %q, %v", got, err)
}
}
func TestBuildPageQuery_CombinedFilters(t *testing.T) {
min, max := int64(10), int64(20)
lo, hi, eq := 1.5, 9.0, "nord"
q, args, err := BuildPageQuery(SearchRequest{
Term: "x",
Filters: []Filter{{Field: FilterCreatedBy, Value: "u1"}},
Facet: FacetFilters{
DocumentTypes: []string{"a", "b"},
Tags: []string{"t1", "t2"},
Ranges: []Range{{Field: RangeUpdatedAt, Min: &min, Max: &max}},
Custom: []CustomFilter{{Key: "betrag", Min: &lo, Max: &hi}, {Key: "region", Eq: &eq}},
},
})
if err != nil {
t.Fatal(err)
}
for _, want := range []string{
"created_by = ?", "document_type IN (?, ?)", "attrs.tags = ? AND attrs.tags = ?",
"updated_at >= ? AND updated_at <= ?", "attrs.f.betrag >= ? AND attrs.f.betrag <= ?", "attrs.fs.region = ?",
} {
if !strings.Contains(q, want) {
t.Errorf("query enthaelt %q nicht: %s", want, q)
}
}
if len(args) != 11 {
t.Errorf("args = %d, want 11 (%v)", len(args), args)
}
for _, a := range args {
if s, ok := a.(string); ok && strings.Contains(q, s) && len(s) > 3 {
t.Errorf("nutzerwert %q steht im sql-text", s)
}
}
}
func TestBuildFacetQuery_RejectsBadInput(t *testing.T) {
req := SearchRequest{Term: "x"}
if _, _, err := BuildFacetQuery(req, nil); !errors.Is(err, ErrInvalidRequest) {
t.Errorf("keine facetten: %v", err)
}
if _, _, err := BuildFacetQuery(req, []FacetSpec{{Kind: FacetCustom, Key: "x; drop"}}); !errors.Is(err, ErrInvalidRequest) {
t.Errorf("boeser key: %v", err)
}
many := make([]string, MaxFilterEntries+1)
if _, _, err := BuildFacetQuery(SearchRequest{Term: "x", Facet: FacetFilters{Tags: many}}, []FacetSpec{{Kind: FacetTag}}); !errors.Is(err, ErrInvalidRequest) {
t.Errorf("zu viele tags: %v", err)
}
}
+259
View File
@@ -0,0 +1,259 @@
package searchindex
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"strconv"
)
const (
MaxFacetValues = 50
MaxFilterEntries = 20
)
// Attrs ist der Inhalt der JSON-Spalte attrs: Tag-Namen, numerische bzw.
// Datumswerte (Epoche) und Rohwerte benutzerdefinierter Felder.
type Attrs struct {
Tags []string `json:"tags"`
Typed map[string]float64 `json:"f"`
Strings map[string]string `json:"fs"`
}
func (a Attrs) JSON() (string, error) {
if a.Tags == nil {
a.Tags = []string{}
}
if a.Typed == nil {
a.Typed = map[string]float64{}
}
if a.Strings == nil {
a.Strings = map[string]string{}
}
b, err := json.Marshal(a)
if err != nil {
return "", fmt.Errorf("searchindex: attrs serialisieren: %w", err)
}
return string(b), nil
}
type RangeField int
const (
RangeCreatedAt RangeField = iota
RangeUpdatedAt
)
// Range ist ein geschlossenes Zeitintervall in Epochensekunden; nil = offen.
type Range struct {
Field RangeField
Min *int64
Max *int64
}
// CustomFilter filtert ein benutzerdefiniertes Feld: Eq gegen den Rohwert,
// Min/Max gegen den numerischen bzw. Datumswert.
type CustomFilter struct {
Key string
Eq *string
Min *float64
Max *float64
}
type FacetKind int
const (
FacetDocumentType FacetKind = iota
FacetTag
FacetCreatedBy
FacetCustom
)
type FacetSpec struct {
Kind FacetKind
Key string
}
// FacetFilters sind die kombinierbaren Filter jenseits der Einzel-Gleichheit:
// DocumentTypes = ODER, Tags/Ranges/Custom = UND.
type FacetFilters struct {
DocumentTypes []string
Tags []string
Ranges []Range
Custom []CustomFilter
}
type FacetValue struct {
Value string
Count int
}
type FacetResult struct {
Spec FacetSpec
Values []FacetValue
}
func buildWhere(req SearchRequest) (where string, args []any, err error) {
where = "MATCH(?)"
args = []any{escapeMatchTerm(req.Term)}
for _, f := range req.Filters {
clause, err := f.Field.clause()
if err != nil {
return "", nil, err
}
where += " AND " + clause
args = append(args, f.Value)
}
fl := req.Facet
if len(fl.DocumentTypes) > MaxFilterEntries || len(fl.Tags) > MaxFilterEntries ||
len(fl.Ranges) > MaxFilterEntries || len(fl.Custom) > MaxFilterEntries {
return "", nil, fmt.Errorf("%w: hoechstens %d werte je filterart", ErrInvalidRequest, MaxFilterEntries)
}
if len(fl.DocumentTypes) > 0 {
where += " AND document_type IN ("
for i, v := range fl.DocumentTypes {
if i > 0 {
where += ", "
}
where += "?"
args = append(args, v)
}
where += ")"
}
for _, tag := range fl.Tags {
where += " AND attrs.tags = ?"
args = append(args, tag)
}
for _, r := range fl.Ranges {
lo, hi, err := r.Field.clauses()
if err != nil {
return "", nil, err
}
if r.Min != nil {
where += " AND " + lo
args = append(args, *r.Min)
}
if r.Max != nil {
where += " AND " + hi
args = append(args, *r.Max)
}
}
for _, c := range fl.Custom {
if c.Eq != nil {
path, err := customPath(pathString, c.Key)
if err != nil {
return "", nil, err
}
where += " AND " + path + " = ?"
args = append(args, *c.Eq)
}
if c.Min != nil || c.Max != nil {
path, err := customPath(pathTyped, c.Key)
if err != nil {
return "", nil, err
}
if c.Min != nil {
where += " AND " + path + " >= ?"
args = append(args, *c.Min)
}
if c.Max != nil {
where += " AND " + path + " <= ?"
args = append(args, *c.Max)
}
}
}
return where, args, nil
}
func (f RangeField) clauses() (lo, hi string, err error) {
switch f {
case RangeCreatedAt:
return "created_at >= ?", "created_at <= ?", nil
case RangeUpdatedAt:
return "updated_at >= ?", "updated_at <= ?", nil
default:
return "", "", fmt.Errorf("%w: unbekanntes bereichsfeld %d", ErrInvalidRequest, f)
}
}
func (s FacetSpec) clause() (string, error) {
switch s.Kind {
case FacetDocumentType:
return " FACET document_type ORDER BY COUNT(*) DESC LIMIT " + strconv.Itoa(MaxFacetValues), nil
case FacetTag:
return " FACET attrs.tags ORDER BY COUNT(*) DESC LIMIT " + strconv.Itoa(MaxFacetValues), nil
case FacetCreatedBy:
return " FACET created_by ORDER BY COUNT(*) DESC LIMIT " + strconv.Itoa(MaxFacetValues), nil
case FacetCustom:
path, err := customPath(pathString, s.Key)
if err != nil {
return "", err
}
return " FACET " + path + " ORDER BY COUNT(*) DESC LIMIT " + strconv.Itoa(MaxFacetValues), nil
default:
return "", fmt.Errorf("%w: unbekannte facette %d", ErrInvalidRequest, s.Kind)
}
}
func BuildFacetQuery(req SearchRequest, specs []FacetSpec) (query string, args []any, err error) {
if len(specs) == 0 || len(specs) > MaxFilterEntries {
return "", nil, fmt.Errorf("%w: 1 bis %d facetten erforderlich", ErrInvalidRequest, MaxFilterEntries)
}
where, args, err := buildWhere(req)
if err != nil {
return "", nil, err
}
query = "SELECT id FROM " + TableName + " WHERE " + where + " LIMIT 1"
for _, s := range specs {
clause, err := s.clause()
if err != nil {
return "", nil, err
}
query += clause
}
return query, args, nil
}
// Facets liefert je Spezifikation die Trefferzahlen je Wert unter allen
// aktiven Filtern; leere/fehlende Werte werden ausgelassen.
func Facets(ctx context.Context, db *sql.DB, req SearchRequest, specs []FacetSpec) ([]FacetResult, error) {
query, args, err := BuildFacetQuery(req, specs)
if err != nil {
return nil, err
}
conn, err := db.Conn(ctx)
if err != nil {
return nil, fmt.Errorf("searchindex: verbindung: %w", err)
}
defer conn.Close()
rows, err := conn.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("searchindex: facetten: %w", err)
}
defer rows.Close()
for rows.Next() {
}
out := make([]FacetResult, 0, len(specs))
for _, spec := range specs {
if !rows.NextResultSet() {
return nil, fmt.Errorf("searchindex: facettenergebnis fehlt: %w", rows.Err())
}
res := FacetResult{Spec: spec, Values: []FacetValue{}}
for rows.Next() {
var v sql.NullString
var n int
if err := rows.Scan(&v, &n); err != nil {
return nil, fmt.Errorf("searchindex: facette lesen: %w", err)
}
if v.Valid && v.String != "" {
res.Values = append(res.Values, FacetValue{Value: v.String, Count: n})
}
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("searchindex: facette lesen: %w", err)
}
out = append(out, res)
}
return out, nil
}
+12 -12
View File
@@ -199,15 +199,20 @@ type Doc struct {
CreatedBy string
CreatedAtSec int64
UpdatedAtSec int64
Attrs Attrs
}
// Upsert legt ein Dokument im Index an oder ersetzt es (REPLACE INTO ist
// bei RT-Tabellen der Upsert-Mechanismus ueber die id).
func Upsert(ctx context.Context, db *sql.DB, d Doc) error {
const query = "REPLACE INTO " + TableName +
" (id, document_id, title, body, tags, folder_path, document_type, created_by, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?)"
_, err := db.ExecContext(ctx, query,
d.ID, d.DocumentID, d.Title, d.Body, d.Tags, d.FolderPath, d.DocumentType, d.CreatedBy, d.CreatedAtSec, d.UpdatedAtSec,
" (id, document_id, title, body, tags, folder_path, document_type, created_by, created_at, updated_at, attrs) VALUES (?,?,?,?,?,?,?,?,?,?,?)"
attrs, err := d.Attrs.JSON()
if err != nil {
return err
}
_, err = db.ExecContext(ctx, query,
d.ID, d.DocumentID, d.Title, d.Body, d.Tags, d.FolderPath, d.DocumentType, d.CreatedBy, d.CreatedAtSec, d.UpdatedAtSec, attrs,
)
if err != nil {
return fmt.Errorf("searchindex: dokument indexieren: %w", err)
@@ -266,6 +271,7 @@ type SearchRequest struct {
Sort Sort
Limit int
Offset int
Facet FacetFilters
}
type PageHit struct {
@@ -325,15 +331,9 @@ func BuildPageQuery(req SearchRequest) (query string, args []any, err error) {
if err != nil {
return "", nil, err
}
where := "MATCH(?)"
args = []any{escapeMatchTerm(req.Term)}
for _, f := range req.Filters {
clause, err := f.Field.clause()
if err != nil {
return "", nil, err
}
where += " AND " + clause
args = append(args, f.Value)
where, args, err := buildWhere(req)
if err != nil {
return "", nil, err
}
query = "SELECT id, document_id, title, folder_path, document_type, updated_at, WEIGHT() AS score, " +
"SNIPPET(body, QUERY(), 'limit=240', 'around=8', 'before_match=" + snippetOpen + "', 'after_match=" + snippetClose + "') AS snippet " +
+34 -1
View File
@@ -30,7 +30,8 @@ const schemaDDL = `CREATE TABLE IF NOT EXISTS ` + TableName + ` (
document_type string attribute,
created_by string attribute,
created_at timestamp,
updated_at timestamp
updated_at timestamp,
attrs json
)
morphology='libstemmer_de'
charset_table='non_cjk, U+00DF, U+00E4, U+00F6, U+00FC, U+00C4, U+00D6, U+00DC'
@@ -65,6 +66,38 @@ func EnsureSchema(ctx context.Context, db *sql.DB) error {
if _, err := db.ExecContext(ctx, schemaDDL); err != nil {
return fmt.Errorf("searchindex: schema anlegen: %w", err)
}
return ensureAttrsColumn(ctx, db)
}
// ensureAttrsColumn ruestet Indizes nach, die vor SRC-08 ohne attrs
// angelegt wurden; bestehende Dokumente erhalten attrs erst per Reindex.
func ensureAttrsColumn(ctx context.Context, db *sql.DB) error {
rows, err := db.QueryContext(ctx, "DESCRIBE "+TableName)
if err != nil {
return fmt.Errorf("searchindex: schema lesen: %w", err)
}
found := false
for rows.Next() {
var field, typ, props sql.NullString
if err := rows.Scan(&field, &typ, &props); err != nil {
_ = rows.Close()
return fmt.Errorf("searchindex: schema lesen: %w", err)
}
if field.String == "attrs" {
found = true
}
}
err = rows.Err()
_ = rows.Close()
if err != nil {
return fmt.Errorf("searchindex: schema lesen: %w", err)
}
if found {
return nil
}
if _, err := db.ExecContext(ctx, "ALTER TABLE "+TableName+" ADD COLUMN attrs json"); err != nil {
return fmt.Errorf("searchindex: attrs-spalte anlegen: %w", err)
}
return nil
}
+2 -2
View File
@@ -17,10 +17,10 @@ func itoa(i int) string { return strconv.Itoa(i) }
func TestQueryBuilder_NoDynamicSQLAssembly(t *testing.T) {
allowedConsts := map[string]bool{
"TableName": true, "snippetOpen": true, "snippetClose": true, "MaxWindow": true,
"where": true, "order": true, "clause": true,
"where": true, "order": true, "clause": true, "path": true, "lo": true, "hi": true,
}
fset := token.NewFileSet()
files := []string{"query.go", "schema.go"}
files := []string{"query.go", "schema.go", "facets.go"}
for _, name := range files {
f, err := parser.ParseFile(fset, name, nil, 0)
if err != nil {
+53 -5
View File
@@ -12,7 +12,9 @@ import (
"errors"
"fmt"
"hash/fnv"
"strconv"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
@@ -126,7 +128,12 @@ func (s *Syncer) SyncDocument(ctx context.Context, documentID string) error {
return searchindex.Delete(ctx, s.Index, id)
}
tags, err := s.loadTags(ctx, documentID)
tagNames, err := s.loadTagNames(ctx, documentID)
if err != nil {
return err
}
tags := strings.Join(tagNames, " ")
attrs, err := s.loadAttrs(ctx, documentID, tagNames)
if err != nil {
return err
}
@@ -151,6 +158,7 @@ func (s *Syncer) SyncDocument(ctx context.Context, documentID string) error {
CreatedBy: createdBy,
CreatedAtSec: createdAtSec,
UpdatedAtSec: updatedAtSec,
Attrs: attrs,
}
if err := searchindex.Upsert(ctx, s.Index, doc); err != nil {
return err
@@ -158,23 +166,63 @@ func (s *Syncer) SyncDocument(ctx context.Context, documentID string) error {
return nil
}
func (s *Syncer) loadTags(ctx context.Context, documentID string) (string, error) {
func (s *Syncer) loadTagNames(ctx context.Context, documentID string) ([]string, error) {
rows, err := s.Pool.Query(ctx, `
SELECT t.name FROM document_tags dt JOIN tags t ON t.id = dt.tag_id WHERE dt.document_id = $1 ORDER BY t.name
`, documentID)
if err != nil {
return "", fmt.Errorf("searchsync: tags lesen: %w", err)
return nil, fmt.Errorf("searchsync: tags lesen: %w", err)
}
defer rows.Close()
var names []string
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return "", fmt.Errorf("searchsync: tag-zeile lesen: %w", err)
return nil, fmt.Errorf("searchsync: tag-zeile lesen: %w", err)
}
names = append(names, name)
}
return strings.Join(names, " "), rows.Err()
return names, rows.Err()
}
func (s *Syncer) loadAttrs(ctx context.Context, documentID string, tagNames []string) (searchindex.Attrs, error) {
attrs := searchindex.Attrs{Tags: tagNames, Typed: map[string]float64{}, Strings: map[string]string{}}
rows, err := s.Pool.Query(ctx, `
SELECT mf.field_key, mf.field_type, v.value
FROM document_metadata_values v JOIN metadata_fields mf ON mf.id = v.field_id
WHERE v.document_id = $1
`, documentID)
if err != nil {
return attrs, fmt.Errorf("searchsync: metadatenwerte lesen: %w", err)
}
defer rows.Close()
for rows.Next() {
var key, typ, value string
if err := rows.Scan(&key, &typ, &value); err != nil {
return attrs, fmt.Errorf("searchsync: metadatenwert lesen: %w", err)
}
if !searchindex.ValidCustomKey(key) {
continue
}
attrs.Strings[key] = value
switch typ {
case "number":
if n, err := strconv.ParseFloat(value, 64); err == nil {
attrs.Typed[key] = n
}
case "date":
if d, err := time.Parse("2006-01-02", value); err == nil {
attrs.Typed[key] = float64(d.Unix())
}
case "bool":
if b, err := strconv.ParseBool(value); err == nil && b {
attrs.Typed[key] = 1
} else if err == nil {
attrs.Typed[key] = 0
}
}
}
return attrs, rows.Err()
}
func (s *Syncer) loadFolderPath(ctx context.Context, folderID *string) (string, error) {