SRC-08: facetten- & filter-api
CI / Core-Modul (Build/Vet/Test) (push) Canceled after 0s
CI / DMS Security-Test-Gate (AC5, Merge-Gate) (push) Canceled after 0s
CI / DMS-Modul (Lint/Build/Test/Coverage/Artefakt) (push) Canceled after 0s
CI / DMS-Modul Supply-Chain-Scan (govulncheck) (push) Canceled after 0s
CI / Web (dms-browser) (push) Canceled after 0s
CI / Web (pdf-viewer) (push) Canceled after 0s
CI / Web (shl) (push) Canceled after 0s
CI / Core-Modul (Build/Vet/Test) (push) Canceled after 0s
CI / DMS Security-Test-Gate (AC5, Merge-Gate) (push) Canceled after 0s
CI / DMS-Modul (Lint/Build/Test/Coverage/Artefakt) (push) Canceled after 0s
CI / DMS-Modul Supply-Chain-Scan (govulncheck) (push) Canceled after 0s
CI / Web (dms-browser) (push) Canceled after 0s
CI / Web (pdf-viewer) (push) Canceled after 0s
CI / Web (shl) (push) Canceled after 0s
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f45cf29184
commit
2a49782510
11 files changed
+1013
-25
No files matched your search
@@ -7,6 +7,10 @@
|
||||
-- in internal/searchindex.BuildSearchQuery statt hier im Schema:
|
||||
-- title = 10, tags = 5, body = 1
|
||||
--
|
||||
-- attrs (SRC-08): JSON mit tags[] (Tag-Namen), f{} (numerische/Datums-
|
||||
-- Werte benutzerdefinierter Felder, Datum als Epoche) und fs{} (Rohwert je
|
||||
-- Feld als String) fuer Facetten und Filter.
|
||||
--
|
||||
-- Deutsche Sprachbehandlung (Akzeptanzkriterium 3):
|
||||
-- morphology='libstemmer_de' -- Snowball-Stemmer fuer Deutsch
|
||||
-- charset_table erweitert um Umlaute/ß als Buchstaben (sonst als
|
||||
@@ -25,7 +29,8 @@ CREATE TABLE IF NOT EXISTS dms_documents (
|
||||
document_type string attribute,
|
||||
created_by string attribute,
|
||||
created_at timestamp,
|
||||
updated_at timestamp
|
||||
updated_at timestamp,
|
||||
attrs json
|
||||
)
|
||||
morphology='libstemmer_de'
|
||||
charset_table='non_cjk, U+00DF, U+00E4, U+00F6, U+00FC, U+00C4, U+00D6, U+00DC'
|
||||
|
||||
@@ -0,0 +1,351 @@
|
||||
package searchapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.perlbach24.de/scripte/nexarch/dms/internal/documenttype"
|
||||
"gitea.perlbach24.de/scripte/nexarch/dms/internal/metadatafield"
|
||||
"gitea.perlbach24.de/scripte/nexarch/dms/internal/searchindex"
|
||||
"gitea.perlbach24.de/scripte/nexarch/dms/internal/searchsync"
|
||||
"gitea.perlbach24.de/scripte/nexarch/dms/internal/tag"
|
||||
"gitea.perlbach24.de/scripte/nexarch/dms/internal/testenv"
|
||||
)
|
||||
|
||||
func facetMap(vals []facetValueJSON) map[string]int {
|
||||
m := map[string]int{}
|
||||
for _, v := range vals {
|
||||
m[v.Value] = v.Count
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func docIDs(out responseJSON) []string {
|
||||
var ids []string
|
||||
for _, h := range out.Hits {
|
||||
ids = append(ids, h.DocumentID)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
func attrsOf(tags []string, typed map[string]float64, strs map[string]string) searchindex.Attrs {
|
||||
return searchindex.Attrs{Tags: tags, Typed: typed, Strings: strs}
|
||||
}
|
||||
|
||||
func TestFacets_MultipleFiltersCombine(t *testing.T) {
|
||||
srv, db := setup(t, denyList{})
|
||||
tok := token(t)
|
||||
now := time.Now().Unix()
|
||||
a := index(t, db, searchindex.Doc{Title: tok + " a", CreatedBy: "u1", DocumentType: "Rechnung", UpdatedAtSec: now,
|
||||
Attrs: attrsOf([]string{"steuer", "2026"}, map[string]float64{"betrag": 100}, map[string]string{"region": "nord"})})
|
||||
index(t, db, searchindex.Doc{Title: tok + " b", CreatedBy: "u1", DocumentType: "Rechnung", UpdatedAtSec: now,
|
||||
Attrs: attrsOf([]string{"steuer"}, map[string]float64{"betrag": 900}, map[string]string{"region": "nord"})})
|
||||
index(t, db, searchindex.Doc{Title: tok + " c", CreatedBy: "u1", DocumentType: "Vertrag", UpdatedAtSec: now,
|
||||
Attrs: attrsOf([]string{"steuer", "2026"}, map[string]float64{"betrag": 100}, map[string]string{"region": "sued"})})
|
||||
|
||||
base := srv.URL + "/search?caller=u1&q=" + tok
|
||||
cases := []struct {
|
||||
name string
|
||||
query string
|
||||
want int
|
||||
}{
|
||||
{"nur typ", "&document_type=Rechnung", 2},
|
||||
{"typ+tag", "&document_type=Rechnung&tag=2026", 1},
|
||||
{"typ+tag+feldbereich", "&document_type=Rechnung&tag=2026&field.betrag.min=50&field.betrag.max=150", 1},
|
||||
{"zwei tags", "&tag=steuer&tag=2026", 2},
|
||||
{"feld-gleichheit+typ", "&field.region=nord&document_type=Rechnung", 2},
|
||||
{"typ-oder", "&document_type=Rechnung&document_type=Vertrag", 3},
|
||||
{"widerspruch", "&document_type=Vertrag&field.region=nord", 0},
|
||||
}
|
||||
for _, c := range cases {
|
||||
status, out := get(t, base+c.query)
|
||||
if status != 200 || out.Total != c.want || len(out.Hits) != c.want {
|
||||
t.Errorf("%s: status=%d total=%d hits=%d, want %d", c.name, status, out.Total, len(out.Hits), c.want)
|
||||
}
|
||||
}
|
||||
_, out := get(t, base+"&document_type=Rechnung&tag=2026&field.betrag.max=150")
|
||||
if ids := docIDs(out); len(ids) != 1 || ids[0] != a.DocumentID {
|
||||
t.Errorf("falsches dokument: %v", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFacets_CountsPerValueAndUnderActiveFilters(t *testing.T) {
|
||||
srv, db := setup(t, denyList{})
|
||||
tok := token(t)
|
||||
now := time.Now().Unix()
|
||||
for i, spec := range []struct {
|
||||
typ string
|
||||
tags []string
|
||||
reg string
|
||||
}{
|
||||
{"Rechnung", []string{"a", "b"}, "nord"}, {"Rechnung", []string{"a"}, "sued"}, {"Rechnung", nil, "nord"},
|
||||
{"Vertrag", []string{"b"}, "nord"}, {"", []string{"a"}, ""},
|
||||
} {
|
||||
index(t, db, searchindex.Doc{Title: fmt.Sprintf("%s %d", tok, i), CreatedBy: "u1", DocumentType: spec.typ, UpdatedAtSec: now,
|
||||
Attrs: attrsOf(spec.tags, nil, map[string]string{"region": spec.reg})})
|
||||
}
|
||||
base := srv.URL + "/search?caller=u1&q=" + tok + "&facets=document_type,tag,created_by,field.region"
|
||||
_, out := get(t, base)
|
||||
if out.Total != 5 {
|
||||
t.Fatalf("total = %d", out.Total)
|
||||
}
|
||||
if got := facetMap(out.Facets["document_type"]); got["Rechnung"] != 3 || got["Vertrag"] != 1 || len(got) != 2 {
|
||||
t.Errorf("document_type = %v", got)
|
||||
}
|
||||
if got := facetMap(out.Facets["tag"]); got["a"] != 3 || got["b"] != 2 || len(got) != 2 {
|
||||
t.Errorf("tag = %v", got)
|
||||
}
|
||||
if got := facetMap(out.Facets["created_by"]); got["u1"] != 5 || len(got) != 1 {
|
||||
t.Errorf("created_by = %v", got)
|
||||
}
|
||||
if got := facetMap(out.Facets["field.region"]); got["nord"] != 3 || got["sued"] != 1 || len(got) != 2 {
|
||||
t.Errorf("field.region = %v", got)
|
||||
}
|
||||
_, out = get(t, base+"&tag=a")
|
||||
if got := facetMap(out.Facets["document_type"]); got["Rechnung"] != 2 || len(got) != 1 {
|
||||
t.Errorf("document_type unter tag=a = %v", got)
|
||||
}
|
||||
if got := facetMap(out.Facets["tag"]); got["a"] != 3 || got["b"] != 1 {
|
||||
t.Errorf("tag unter tag=a = %v", got)
|
||||
}
|
||||
if out.Facets["tag"][0].Value != "a" {
|
||||
t.Errorf("facetten nicht nach anzahl absteigend: %v", out.Facets["tag"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilters_DateAndValueRanges(t *testing.T) {
|
||||
srv, db := setup(t, denyList{})
|
||||
tok := token(t)
|
||||
day := func(s string) int64 {
|
||||
d, _ := time.Parse("2006-01-02", s)
|
||||
return d.Unix() + 3600
|
||||
}
|
||||
for i, spec := range []struct {
|
||||
updated string
|
||||
betrag float64
|
||||
frist string
|
||||
}{
|
||||
{"2026-01-10", 10, "2026-03-01"}, {"2026-02-10", 20, "2026-04-01"}, {"2026-03-10", 30, "2026-05-01"},
|
||||
} {
|
||||
frist, _ := time.Parse("2006-01-02", spec.frist)
|
||||
index(t, db, searchindex.Doc{Title: fmt.Sprintf("%s %d", tok, i), CreatedBy: "u1", UpdatedAtSec: day(spec.updated),
|
||||
Attrs: attrsOf(nil, map[string]float64{"betrag": spec.betrag, "frist": float64(frist.Unix())}, nil)})
|
||||
}
|
||||
base := srv.URL + "/search?caller=u1&q=" + tok
|
||||
for _, c := range []struct {
|
||||
query string
|
||||
want int
|
||||
}{
|
||||
{"&updated_from=2026-02-01", 2},
|
||||
{"&updated_to=2026-02-10", 2},
|
||||
{"&updated_from=2026-02-10&updated_to=2026-02-10", 1},
|
||||
{"&updated_from=2026-01-15&updated_to=2026-03-01", 1},
|
||||
{"&created_from=2026-01-01T00:00:00Z", 3},
|
||||
{"&field.betrag.min=15&field.betrag.max=25", 1},
|
||||
{"&field.betrag.min=20", 2},
|
||||
{"&field.frist.min=2026-04-01&field.frist.max=2026-04-30", 1},
|
||||
{"&field.frist.max=2026-03-31", 1},
|
||||
} {
|
||||
if status, out := get(t, base+c.query); status != 200 || out.Total != c.want {
|
||||
t.Errorf("%s: status=%d total=%d, want %d", c.query, status, out.Total, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilters_InvalidParametersRejected(t *testing.T) {
|
||||
srv, _ := setup(t, denyList{})
|
||||
base := srv.URL + "/search?caller=u1&q=x"
|
||||
for _, q := range []string{
|
||||
"&updated_from=gestern", "&created_from=2026-03-01&created_to=2026-01-01", "&field.Betrag=x", "&field.a.b.min=1",
|
||||
"&field.betrag.min=abc", "&facets=passwort", "&facets=field.a;drop", "&field.x%3Bdrop=1", "&field.betrag.avg=1",
|
||||
} {
|
||||
if status, _ := get(t, base+q); status != 400 {
|
||||
t.Errorf("%s: status = %d, want 400", q, status)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pruefung 2: leeres Ergebnis bleibt stabil (Struktur, Facetten leer, kein Fehler).
|
||||
func TestFilters_EmptyResultIsStable(t *testing.T) {
|
||||
srv, db := setup(t, denyList{})
|
||||
tok := token(t)
|
||||
index(t, db, searchindex.Doc{Title: tok, CreatedBy: "u1", DocumentType: "Rechnung"})
|
||||
status, out := get(t, srv.URL+"/search?caller=u1&q="+tok+"&document_type=Nichts&tag=x&field.a.min=1&facets=document_type,tag,field.a")
|
||||
if status != 200 || out.Total != 0 || out.Hits == nil || len(out.Hits) != 0 {
|
||||
t.Fatalf("status=%d total=%d hits=%v", status, out.Total, out.Hits)
|
||||
}
|
||||
for _, name := range []string{"document_type", "tag", "field.a"} {
|
||||
vals, ok := out.Facets[name]
|
||||
if !ok || len(vals) != 0 {
|
||||
t.Errorf("facette %s = %v (vorhanden=%v), want leer", name, vals, ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pruefung 3: Antwortzeit mit fuenf aktiven Filtern.
|
||||
func TestFilters_ResponseTimeWithFiveFilters(t *testing.T) {
|
||||
srv, db := setup(t, denyList{})
|
||||
tok := token(t)
|
||||
now := time.Now().Unix()
|
||||
for i := 0; i < 200; i++ {
|
||||
index(t, db, searchindex.Doc{Title: fmt.Sprintf("%s %d", tok, i), CreatedBy: "u1", DocumentType: "Rechnung", UpdatedAtSec: now,
|
||||
Attrs: attrsOf([]string{"steuer", "2026"}, map[string]float64{"betrag": float64(i)}, map[string]string{"region": "nord"})})
|
||||
}
|
||||
q := url.Values{"caller": {"u1"}, "q": {tok}, "document_type": {"Rechnung"}, "tag": {"steuer", "2026"},
|
||||
"updated_from": {"2020-01-01"}, "field.betrag.min": {"10"}, "field.region": {"nord"}, "facets": {"document_type,tag,field.region"}}
|
||||
var durations []time.Duration
|
||||
for i := 0; i < 30; i++ {
|
||||
start := time.Now()
|
||||
status, out := get(t, srv.URL+"/search?"+q.Encode())
|
||||
durations = append(durations, time.Since(start))
|
||||
if status != 200 || out.Total != 190 {
|
||||
t.Fatalf("status=%d total=%d", status, out.Total)
|
||||
}
|
||||
}
|
||||
sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] })
|
||||
p95 := durations[len(durations)*95/100]
|
||||
t.Logf("5 filter + 3 facetten: median=%v p95=%v max=%v", durations[len(durations)/2], p95, durations[len(durations)-1])
|
||||
if p95 > responseTimeLimit {
|
||||
t.Errorf("p95 = %v ueber %v", p95, responseTimeLimit)
|
||||
}
|
||||
}
|
||||
|
||||
// Pruefung 1: Facetten- und Trefferzahlen gegen SQL auf der Quelldatenbank abgeglichen.
|
||||
func TestFacets_CountsMatchDatabaseQuery(t *testing.T) {
|
||||
if os.Getenv("TEST_TENANT_DSN") == "" {
|
||||
t.Skip("TEST_TENANT_DSN nicht gesetzt")
|
||||
}
|
||||
srv, idx := setup(t, denyList{})
|
||||
e := testenv.New(t)
|
||||
ctx := context.Background()
|
||||
syncer := &searchsync.Syncer{Pool: e.Pool, Index: idx}
|
||||
tok := token(t)
|
||||
|
||||
types := map[string]string{}
|
||||
for _, n := range []string{"Rechnung", "Vertrag"} {
|
||||
dt, err := documenttype.Create(ctx, e.Pool, n, nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
types[n] = dt.ID
|
||||
}
|
||||
tags := map[string]string{}
|
||||
for _, n := range []string{"steuer", "2026", "intern"} {
|
||||
tg, err := tag.Create(ctx, e.Pool, n, "", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tags[n] = tg.ID
|
||||
}
|
||||
fields := map[string]string{}
|
||||
for key, typ := range map[string]string{"betrag": "number", "region": "select", "frist": "date"} {
|
||||
f, err := metadatafield.Create(ctx, e.Pool, key, key, typ, false, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("feld %s: %v", key, err)
|
||||
}
|
||||
fields[key] = f.ID
|
||||
}
|
||||
|
||||
typeNames := []string{"Rechnung", "Vertrag"}
|
||||
tagNames := []string{"steuer", "2026", "intern"}
|
||||
regions := []string{"nord", "sued", "west"}
|
||||
const n = 36
|
||||
for i := 0; i < n; i++ {
|
||||
id, _ := e.UploadDoc(t, fmt.Sprintf("%s-%02d.txt", tok, i), "text/plain", []byte(fmt.Sprintf("inhalt %s %d", tok, i)))
|
||||
t.Cleanup(func() { _ = searchindex.Delete(ctx, idx, searchsync.DocIndexID(id)) })
|
||||
if _, err := documenttype.ChangeDocumentType(ctx, e.Pool, id, types[typeNames[i%2]], false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for j, tn := range tagNames {
|
||||
if (i>>j)&1 == 1 || (i%5 == 0 && j == 0) {
|
||||
if err := tag.Assign(ctx, e.Pool, id, tags[tn]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for key, val := range map[string]string{
|
||||
"betrag": fmt.Sprintf("%d", (i*37)%500), "region": regions[i%3],
|
||||
"frist": time.Date(2026, time.Month(1+i%6), 1+i%27, 0, 0, 0, 0, time.UTC).Format("2006-01-02"),
|
||||
} {
|
||||
if err := metadatafield.SetValue(ctx, e.Pool, id, fields[key], val); err != nil {
|
||||
t.Fatalf("wert %s=%s: %v", key, val, err)
|
||||
}
|
||||
}
|
||||
if err := syncer.SyncDocument(ctx, id); err != nil {
|
||||
t.Fatalf("sync: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
count := func(sqlText string, args ...any) int {
|
||||
t.Helper()
|
||||
var c int
|
||||
if err := e.Pool.QueryRow(ctx, sqlText, args...).Scan(&c); err != nil {
|
||||
t.Fatalf("sql: %v", err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
facetsQ := "&facets=document_type,tag,field.region"
|
||||
filterQ := "&document_type=Rechnung&tag=steuer&field.betrag.min=100&field.betrag.max=400&field.frist.min=2026-02-01&field.frist.max=2026-05-31"
|
||||
const filterSQL = `
|
||||
FROM documents d
|
||||
JOIN document_types dt ON dt.id = d.document_type_id
|
||||
WHERE d.title LIKE $1 AND dt.name = 'Rechnung'
|
||||
AND EXISTS (SELECT 1 FROM document_tags x JOIN tags t ON t.id = x.tag_id WHERE x.document_id = d.id AND t.name = 'steuer')
|
||||
AND EXISTS (SELECT 1 FROM document_metadata_values v JOIN metadata_fields f ON f.id = v.field_id
|
||||
WHERE v.document_id = d.id AND f.field_key = 'betrag' AND v.value::numeric BETWEEN 100 AND 400)
|
||||
AND EXISTS (SELECT 1 FROM document_metadata_values v JOIN metadata_fields f ON f.id = v.field_id
|
||||
WHERE v.document_id = d.id AND f.field_key = 'frist' AND v.value::date BETWEEN '2026-02-01' AND '2026-05-31')`
|
||||
like := tok + "%"
|
||||
|
||||
base := srv.URL + "/search?caller=" + e.UserID + "&limit=100&q=" + tok
|
||||
_, all := get(t, base+facetsQ)
|
||||
if want := count(`SELECT count(*) FROM documents WHERE title LIKE $1`, like); all.Total != want || want != n {
|
||||
t.Fatalf("gesamt: index=%d db=%d, erwartet %d", all.Total, want, n)
|
||||
}
|
||||
for _, name := range typeNames {
|
||||
want := count(`SELECT count(*) FROM documents d JOIN document_types dt ON dt.id = d.document_type_id WHERE d.title LIKE $1 AND dt.name = $2`, like, name)
|
||||
if got := facetMap(all.Facets["document_type"])[name]; got != want {
|
||||
t.Errorf("facette typ %s: index=%d db=%d", name, got, want)
|
||||
}
|
||||
}
|
||||
for _, name := range tagNames {
|
||||
want := count(`SELECT count(DISTINCT d.id) FROM documents d JOIN document_tags x ON x.document_id = d.id JOIN tags t ON t.id = x.tag_id WHERE d.title LIKE $1 AND t.name = $2`, like, name)
|
||||
if got := facetMap(all.Facets["tag"])[name]; got != want {
|
||||
t.Errorf("facette tag %s: index=%d db=%d", name, got, want)
|
||||
}
|
||||
}
|
||||
for _, r := range regions {
|
||||
want := count(`SELECT count(*) FROM documents d JOIN document_metadata_values v ON v.document_id = d.id JOIN metadata_fields f ON f.id = v.field_id WHERE d.title LIKE $1 AND f.field_key = 'region' AND v.value = $2`, like, r)
|
||||
if got := facetMap(all.Facets["field.region"])[r]; got != want {
|
||||
t.Errorf("facette region %s: index=%d db=%d", r, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
_, filtered := get(t, base+filterQ+facetsQ)
|
||||
wantTotal := count(`SELECT count(*) `+filterSQL, like)
|
||||
if wantTotal == 0 {
|
||||
t.Fatal("testdaten ergeben leere filtermenge, pruefung sinnlos")
|
||||
}
|
||||
if filtered.Total != wantTotal || len(filtered.Hits) != wantTotal {
|
||||
t.Errorf("gefilterte treffer: index=%d/%d db=%d", filtered.Total, len(filtered.Hits), wantTotal)
|
||||
}
|
||||
for _, r := range regions {
|
||||
want := count(`SELECT count(*) `+filterSQL+` AND EXISTS (SELECT 1 FROM document_metadata_values v JOIN metadata_fields f ON f.id = v.field_id WHERE v.document_id = d.id AND f.field_key = 'region' AND v.value = $2)`, like, r)
|
||||
if got := facetMap(filtered.Facets["field.region"])[r]; got != want {
|
||||
t.Errorf("gefilterte facette region %s: index=%d db=%d", r, got, want)
|
||||
}
|
||||
}
|
||||
if got := facetMap(filtered.Facets["document_type"]); len(got) != 1 || got["Rechnung"] != wantTotal {
|
||||
t.Errorf("gefilterte facette typ = %v, want Rechnung=%d", got, wantTotal)
|
||||
}
|
||||
if strings.TrimSpace(tok) == "" {
|
||||
t.Fatal("token leer")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package searchapi
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.perlbach24.de/scripte/nexarch/dms/internal/searchindex"
|
||||
)
|
||||
|
||||
const fieldPrefix = "field."
|
||||
|
||||
func invalid(format string, a ...any) error {
|
||||
return fmt.Errorf("%w: %s", searchindex.ErrInvalidRequest, fmt.Sprintf(format, a...))
|
||||
}
|
||||
|
||||
func facetName(s searchindex.FacetSpec) string {
|
||||
switch s.Kind {
|
||||
case searchindex.FacetDocumentType:
|
||||
return "document_type"
|
||||
case searchindex.FacetTag:
|
||||
return "tag"
|
||||
case searchindex.FacetCreatedBy:
|
||||
return "created_by"
|
||||
default:
|
||||
return fieldPrefix + s.Key
|
||||
}
|
||||
}
|
||||
|
||||
// parseTime akzeptiert YYYY-MM-DD oder RFC3339; endOfDay macht ein reines
|
||||
// Datum zur letzten Sekunde des Tages (obere Bereichsgrenze).
|
||||
func parseTime(v string, endOfDay bool) (int64, error) {
|
||||
if d, err := time.Parse("2006-01-02", v); err == nil {
|
||||
if endOfDay {
|
||||
d = d.Add(24*time.Hour - time.Second)
|
||||
}
|
||||
return d.Unix(), nil
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339, v)
|
||||
if err != nil {
|
||||
return 0, invalid("zeitwert %q ungueltig (YYYY-MM-DD oder RFC3339)", v)
|
||||
}
|
||||
return t.Unix(), nil
|
||||
}
|
||||
|
||||
func parseBound(v string, endOfDay bool) (float64, error) {
|
||||
if n, err := strconv.ParseFloat(v, 64); err == nil {
|
||||
return n, nil
|
||||
}
|
||||
t, err := parseTime(v, endOfDay)
|
||||
if err != nil {
|
||||
return 0, invalid("bereichswert %q ist weder zahl noch datum", v)
|
||||
}
|
||||
return float64(t), nil
|
||||
}
|
||||
|
||||
func parseFacetParams(q url.Values) (searchindex.FacetFilters, []searchindex.FacetSpec, error) {
|
||||
var ff searchindex.FacetFilters
|
||||
var specs []searchindex.FacetSpec
|
||||
ff.DocumentTypes = nonEmpty(q["document_type"])
|
||||
ff.Tags = nonEmpty(q["tag"])
|
||||
|
||||
for _, r := range []struct {
|
||||
field searchindex.RangeField
|
||||
from, to string
|
||||
}{
|
||||
{searchindex.RangeCreatedAt, "created_from", "created_to"},
|
||||
{searchindex.RangeUpdatedAt, "updated_from", "updated_to"},
|
||||
} {
|
||||
rg := searchindex.Range{Field: r.field}
|
||||
if v := q.Get(r.from); v != "" {
|
||||
t, err := parseTime(v, false)
|
||||
if err != nil {
|
||||
return ff, nil, err
|
||||
}
|
||||
rg.Min = &t
|
||||
}
|
||||
if v := q.Get(r.to); v != "" {
|
||||
t, err := parseTime(v, true)
|
||||
if err != nil {
|
||||
return ff, nil, err
|
||||
}
|
||||
rg.Max = &t
|
||||
}
|
||||
if rg.Min != nil && rg.Max != nil && *rg.Min > *rg.Max {
|
||||
return ff, nil, invalid("%s liegt nach %s", r.from, r.to)
|
||||
}
|
||||
if rg.Min != nil || rg.Max != nil {
|
||||
ff.Ranges = append(ff.Ranges, rg)
|
||||
}
|
||||
}
|
||||
|
||||
custom := map[string]*searchindex.CustomFilter{}
|
||||
var keys []string
|
||||
for name, vals := range q {
|
||||
if !strings.HasPrefix(name, fieldPrefix) || len(vals) == 0 || vals[0] == "" {
|
||||
continue
|
||||
}
|
||||
key, part := strings.TrimPrefix(name, fieldPrefix), ""
|
||||
if i := strings.LastIndex(key, "."); i >= 0 {
|
||||
key, part = key[:i], key[i+1:]
|
||||
if part != "min" && part != "max" {
|
||||
return ff, nil, invalid("parameter %q unbekannt", name)
|
||||
}
|
||||
}
|
||||
if !searchindex.ValidCustomKey(key) {
|
||||
return ff, nil, invalid("feldschluessel %q nicht erlaubt", key)
|
||||
}
|
||||
c := custom[key]
|
||||
if c == nil {
|
||||
c = &searchindex.CustomFilter{Key: key}
|
||||
custom[key] = c
|
||||
keys = append(keys, key)
|
||||
}
|
||||
switch part {
|
||||
case "":
|
||||
v := vals[0]
|
||||
c.Eq = &v
|
||||
case "min":
|
||||
n, err := parseBound(vals[0], false)
|
||||
if err != nil {
|
||||
return ff, nil, err
|
||||
}
|
||||
c.Min = &n
|
||||
case "max":
|
||||
n, err := parseBound(vals[0], true)
|
||||
if err != nil {
|
||||
return ff, nil, err
|
||||
}
|
||||
c.Max = &n
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
ff.Custom = append(ff.Custom, *custom[k])
|
||||
}
|
||||
|
||||
if v := strings.TrimSpace(q.Get("facets")); v != "" {
|
||||
for _, name := range strings.Split(v, ",") {
|
||||
switch name = strings.TrimSpace(name); {
|
||||
case name == "document_type":
|
||||
specs = append(specs, searchindex.FacetSpec{Kind: searchindex.FacetDocumentType})
|
||||
case name == "tag":
|
||||
specs = append(specs, searchindex.FacetSpec{Kind: searchindex.FacetTag})
|
||||
case name == "created_by":
|
||||
specs = append(specs, searchindex.FacetSpec{Kind: searchindex.FacetCreatedBy})
|
||||
case strings.HasPrefix(name, fieldPrefix) && searchindex.ValidCustomKey(strings.TrimPrefix(name, fieldPrefix)):
|
||||
specs = append(specs, searchindex.FacetSpec{Kind: searchindex.FacetCustom, Key: strings.TrimPrefix(name, fieldPrefix)})
|
||||
default:
|
||||
return ff, nil, invalid("facette %q nicht erlaubt", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ff, specs, nil
|
||||
}
|
||||
|
||||
func nonEmpty(in []string) []string {
|
||||
var out []string
|
||||
for _, v := range in {
|
||||
if v != "" {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -48,6 +48,13 @@ type responseJSON struct {
|
||||
Offset int `json:"offset"`
|
||||
TookMS int64 `json:"took_ms"`
|
||||
Hits []hitJSON `json:"hits"`
|
||||
// Facets nur bei gesetztem Parameter "facets"; Schluessel wie dort angegeben.
|
||||
Facets map[string][]facetValueJSON `json:"facets,omitempty"`
|
||||
}
|
||||
|
||||
type facetValueJSON struct {
|
||||
Value string `json:"value"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, message string) {
|
||||
@@ -105,7 +112,7 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
filters := []searchindex.Filter{{Field: searchindex.FilterCreatedBy, Value: caller}}
|
||||
for _, name := range []string{"document_type", "folder_path"} {
|
||||
for _, name := range []string{"folder_path", "created_by"} {
|
||||
if v := q.Get(name); v != "" {
|
||||
field, err := searchindex.ParseFilterField(name)
|
||||
if err != nil {
|
||||
@@ -115,10 +122,16 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
|
||||
filters = append(filters, searchindex.Filter{Field: field, Value: v})
|
||||
}
|
||||
}
|
||||
facetFilters, specs, err := parseFacetParams(q)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
searchReq := searchindex.SearchRequest{
|
||||
Term: term, Filters: filters, Sort: sortVal, Limit: limit, Offset: offset, Facet: facetFilters,
|
||||
}
|
||||
|
||||
page, err := searchindex.SearchPage(r.Context(), s.index, searchindex.SearchRequest{
|
||||
Term: term, Filters: filters, Sort: sortVal, Limit: limit, Offset: offset,
|
||||
})
|
||||
page, err := searchindex.SearchPage(r.Context(), s.index, searchReq)
|
||||
if err != nil {
|
||||
if errors.Is(err, searchindex.ErrInvalidRequest) {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
@@ -146,6 +159,25 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
|
||||
Score: h.Score, UpdatedAt: h.UpdatedAt, SnippetHTML: renderSnippet(h.Snippet),
|
||||
})
|
||||
}
|
||||
if len(specs) > 0 {
|
||||
results, err := searchindex.Facets(r.Context(), s.index, searchReq, specs)
|
||||
if err != nil {
|
||||
if errors.Is(err, searchindex.ErrInvalidRequest) {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
resp.Facets = map[string][]facetValueJSON{}
|
||||
for _, fr := range results {
|
||||
vals := make([]facetValueJSON, 0, len(fr.Values))
|
||||
for _, v := range fr.Values {
|
||||
vals = append(vals, facetValueJSON{Value: v.Value, Count: v.Count})
|
||||
}
|
||||
resp.Facets[facetName(fr.Spec)] = vals
|
||||
}
|
||||
}
|
||||
resp.TookMS = time.Since(start).Milliseconds()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package searchindex
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// Feldschluessel benutzerdefinierter Felder duerfen als JSON-Pfad in die
|
||||
// Abfrage; deshalb nur dieses enge Alphabet. Diese Datei ist die einzige
|
||||
// Stelle, an der ein Nutzerwert Teil des SQL-Textes wird.
|
||||
var customKeyPattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,62}$`)
|
||||
|
||||
func ValidCustomKey(key string) bool { return customKeyPattern.MatchString(key) }
|
||||
|
||||
const (
|
||||
pathTyped = "attrs.f."
|
||||
pathString = "attrs.fs."
|
||||
)
|
||||
|
||||
func customPath(prefix, key string) (string, error) {
|
||||
if !ValidCustomKey(key) {
|
||||
return "", fmt.Errorf("%w: feldschluessel %q nicht erlaubt", ErrInvalidRequest, key)
|
||||
}
|
||||
return prefix + key, nil
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package searchindex
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCustomPath_RejectsAnythingButPlainKeys(t *testing.T) {
|
||||
for _, key := range []string{"", "Betrag", "1abc", "a b", "a.b", "a;drop", "a'b", "a)or(1=1", "a-b", strings.Repeat("a", 64), "ä"} {
|
||||
if _, err := customPath(pathString, key); !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Errorf("key %q: err = %v, want ErrInvalidRequest", key, err)
|
||||
}
|
||||
}
|
||||
got, err := customPath(pathTyped, "betrag_netto")
|
||||
if err != nil || got != "attrs.f.betrag_netto" {
|
||||
t.Errorf("got %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPageQuery_CombinedFilters(t *testing.T) {
|
||||
min, max := int64(10), int64(20)
|
||||
lo, hi, eq := 1.5, 9.0, "nord"
|
||||
q, args, err := BuildPageQuery(SearchRequest{
|
||||
Term: "x",
|
||||
Filters: []Filter{{Field: FilterCreatedBy, Value: "u1"}},
|
||||
Facet: FacetFilters{
|
||||
DocumentTypes: []string{"a", "b"},
|
||||
Tags: []string{"t1", "t2"},
|
||||
Ranges: []Range{{Field: RangeUpdatedAt, Min: &min, Max: &max}},
|
||||
Custom: []CustomFilter{{Key: "betrag", Min: &lo, Max: &hi}, {Key: "region", Eq: &eq}},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"created_by = ?", "document_type IN (?, ?)", "attrs.tags = ? AND attrs.tags = ?",
|
||||
"updated_at >= ? AND updated_at <= ?", "attrs.f.betrag >= ? AND attrs.f.betrag <= ?", "attrs.fs.region = ?",
|
||||
} {
|
||||
if !strings.Contains(q, want) {
|
||||
t.Errorf("query enthaelt %q nicht: %s", want, q)
|
||||
}
|
||||
}
|
||||
if len(args) != 11 {
|
||||
t.Errorf("args = %d, want 11 (%v)", len(args), args)
|
||||
}
|
||||
for _, a := range args {
|
||||
if s, ok := a.(string); ok && strings.Contains(q, s) && len(s) > 3 {
|
||||
t.Errorf("nutzerwert %q steht im sql-text", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFacetQuery_RejectsBadInput(t *testing.T) {
|
||||
req := SearchRequest{Term: "x"}
|
||||
if _, _, err := BuildFacetQuery(req, nil); !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Errorf("keine facetten: %v", err)
|
||||
}
|
||||
if _, _, err := BuildFacetQuery(req, []FacetSpec{{Kind: FacetCustom, Key: "x; drop"}}); !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Errorf("boeser key: %v", err)
|
||||
}
|
||||
many := make([]string, MaxFilterEntries+1)
|
||||
if _, _, err := BuildFacetQuery(SearchRequest{Term: "x", Facet: FacetFilters{Tags: many}}, []FacetSpec{{Kind: FacetTag}}); !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Errorf("zu viele tags: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package searchindex
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxFacetValues = 50
|
||||
MaxFilterEntries = 20
|
||||
)
|
||||
|
||||
// Attrs ist der Inhalt der JSON-Spalte attrs: Tag-Namen, numerische bzw.
|
||||
// Datumswerte (Epoche) und Rohwerte benutzerdefinierter Felder.
|
||||
type Attrs struct {
|
||||
Tags []string `json:"tags"`
|
||||
Typed map[string]float64 `json:"f"`
|
||||
Strings map[string]string `json:"fs"`
|
||||
}
|
||||
|
||||
func (a Attrs) JSON() (string, error) {
|
||||
if a.Tags == nil {
|
||||
a.Tags = []string{}
|
||||
}
|
||||
if a.Typed == nil {
|
||||
a.Typed = map[string]float64{}
|
||||
}
|
||||
if a.Strings == nil {
|
||||
a.Strings = map[string]string{}
|
||||
}
|
||||
b, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("searchindex: attrs serialisieren: %w", err)
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
type RangeField int
|
||||
|
||||
const (
|
||||
RangeCreatedAt RangeField = iota
|
||||
RangeUpdatedAt
|
||||
)
|
||||
|
||||
// Range ist ein geschlossenes Zeitintervall in Epochensekunden; nil = offen.
|
||||
type Range struct {
|
||||
Field RangeField
|
||||
Min *int64
|
||||
Max *int64
|
||||
}
|
||||
|
||||
// CustomFilter filtert ein benutzerdefiniertes Feld: Eq gegen den Rohwert,
|
||||
// Min/Max gegen den numerischen bzw. Datumswert.
|
||||
type CustomFilter struct {
|
||||
Key string
|
||||
Eq *string
|
||||
Min *float64
|
||||
Max *float64
|
||||
}
|
||||
|
||||
type FacetKind int
|
||||
|
||||
const (
|
||||
FacetDocumentType FacetKind = iota
|
||||
FacetTag
|
||||
FacetCreatedBy
|
||||
FacetCustom
|
||||
)
|
||||
|
||||
type FacetSpec struct {
|
||||
Kind FacetKind
|
||||
Key string
|
||||
}
|
||||
|
||||
// FacetFilters sind die kombinierbaren Filter jenseits der Einzel-Gleichheit:
|
||||
// DocumentTypes = ODER, Tags/Ranges/Custom = UND.
|
||||
type FacetFilters struct {
|
||||
DocumentTypes []string
|
||||
Tags []string
|
||||
Ranges []Range
|
||||
Custom []CustomFilter
|
||||
}
|
||||
|
||||
type FacetValue struct {
|
||||
Value string
|
||||
Count int
|
||||
}
|
||||
|
||||
type FacetResult struct {
|
||||
Spec FacetSpec
|
||||
Values []FacetValue
|
||||
}
|
||||
|
||||
func buildWhere(req SearchRequest) (where string, args []any, err error) {
|
||||
where = "MATCH(?)"
|
||||
args = []any{escapeMatchTerm(req.Term)}
|
||||
for _, f := range req.Filters {
|
||||
clause, err := f.Field.clause()
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
where += " AND " + clause
|
||||
args = append(args, f.Value)
|
||||
}
|
||||
fl := req.Facet
|
||||
if len(fl.DocumentTypes) > MaxFilterEntries || len(fl.Tags) > MaxFilterEntries ||
|
||||
len(fl.Ranges) > MaxFilterEntries || len(fl.Custom) > MaxFilterEntries {
|
||||
return "", nil, fmt.Errorf("%w: hoechstens %d werte je filterart", ErrInvalidRequest, MaxFilterEntries)
|
||||
}
|
||||
if len(fl.DocumentTypes) > 0 {
|
||||
where += " AND document_type IN ("
|
||||
for i, v := range fl.DocumentTypes {
|
||||
if i > 0 {
|
||||
where += ", "
|
||||
}
|
||||
where += "?"
|
||||
args = append(args, v)
|
||||
}
|
||||
where += ")"
|
||||
}
|
||||
for _, tag := range fl.Tags {
|
||||
where += " AND attrs.tags = ?"
|
||||
args = append(args, tag)
|
||||
}
|
||||
for _, r := range fl.Ranges {
|
||||
lo, hi, err := r.Field.clauses()
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if r.Min != nil {
|
||||
where += " AND " + lo
|
||||
args = append(args, *r.Min)
|
||||
}
|
||||
if r.Max != nil {
|
||||
where += " AND " + hi
|
||||
args = append(args, *r.Max)
|
||||
}
|
||||
}
|
||||
for _, c := range fl.Custom {
|
||||
if c.Eq != nil {
|
||||
path, err := customPath(pathString, c.Key)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
where += " AND " + path + " = ?"
|
||||
args = append(args, *c.Eq)
|
||||
}
|
||||
if c.Min != nil || c.Max != nil {
|
||||
path, err := customPath(pathTyped, c.Key)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if c.Min != nil {
|
||||
where += " AND " + path + " >= ?"
|
||||
args = append(args, *c.Min)
|
||||
}
|
||||
if c.Max != nil {
|
||||
where += " AND " + path + " <= ?"
|
||||
args = append(args, *c.Max)
|
||||
}
|
||||
}
|
||||
}
|
||||
return where, args, nil
|
||||
}
|
||||
|
||||
func (f RangeField) clauses() (lo, hi string, err error) {
|
||||
switch f {
|
||||
case RangeCreatedAt:
|
||||
return "created_at >= ?", "created_at <= ?", nil
|
||||
case RangeUpdatedAt:
|
||||
return "updated_at >= ?", "updated_at <= ?", nil
|
||||
default:
|
||||
return "", "", fmt.Errorf("%w: unbekanntes bereichsfeld %d", ErrInvalidRequest, f)
|
||||
}
|
||||
}
|
||||
|
||||
func (s FacetSpec) clause() (string, error) {
|
||||
switch s.Kind {
|
||||
case FacetDocumentType:
|
||||
return " FACET document_type ORDER BY COUNT(*) DESC LIMIT " + strconv.Itoa(MaxFacetValues), nil
|
||||
case FacetTag:
|
||||
return " FACET attrs.tags ORDER BY COUNT(*) DESC LIMIT " + strconv.Itoa(MaxFacetValues), nil
|
||||
case FacetCreatedBy:
|
||||
return " FACET created_by ORDER BY COUNT(*) DESC LIMIT " + strconv.Itoa(MaxFacetValues), nil
|
||||
case FacetCustom:
|
||||
path, err := customPath(pathString, s.Key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return " FACET " + path + " ORDER BY COUNT(*) DESC LIMIT " + strconv.Itoa(MaxFacetValues), nil
|
||||
default:
|
||||
return "", fmt.Errorf("%w: unbekannte facette %d", ErrInvalidRequest, s.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func BuildFacetQuery(req SearchRequest, specs []FacetSpec) (query string, args []any, err error) {
|
||||
if len(specs) == 0 || len(specs) > MaxFilterEntries {
|
||||
return "", nil, fmt.Errorf("%w: 1 bis %d facetten erforderlich", ErrInvalidRequest, MaxFilterEntries)
|
||||
}
|
||||
where, args, err := buildWhere(req)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
query = "SELECT id FROM " + TableName + " WHERE " + where + " LIMIT 1"
|
||||
for _, s := range specs {
|
||||
clause, err := s.clause()
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
query += clause
|
||||
}
|
||||
return query, args, nil
|
||||
}
|
||||
|
||||
// Facets liefert je Spezifikation die Trefferzahlen je Wert unter allen
|
||||
// aktiven Filtern; leere/fehlende Werte werden ausgelassen.
|
||||
func Facets(ctx context.Context, db *sql.DB, req SearchRequest, specs []FacetSpec) ([]FacetResult, error) {
|
||||
query, args, err := BuildFacetQuery(req, specs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("searchindex: verbindung: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
rows, err := conn.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("searchindex: facetten: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
}
|
||||
out := make([]FacetResult, 0, len(specs))
|
||||
for _, spec := range specs {
|
||||
if !rows.NextResultSet() {
|
||||
return nil, fmt.Errorf("searchindex: facettenergebnis fehlt: %w", rows.Err())
|
||||
}
|
||||
res := FacetResult{Spec: spec, Values: []FacetValue{}}
|
||||
for rows.Next() {
|
||||
var v sql.NullString
|
||||
var n int
|
||||
if err := rows.Scan(&v, &n); err != nil {
|
||||
return nil, fmt.Errorf("searchindex: facette lesen: %w", err)
|
||||
}
|
||||
if v.Valid && v.String != "" {
|
||||
res.Values = append(res.Values, FacetValue{Value: v.String, Count: n})
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("searchindex: facette lesen: %w", err)
|
||||
}
|
||||
out = append(out, res)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -199,15 +199,20 @@ type Doc struct {
|
||||
CreatedBy string
|
||||
CreatedAtSec int64
|
||||
UpdatedAtSec int64
|
||||
Attrs Attrs
|
||||
}
|
||||
|
||||
// Upsert legt ein Dokument im Index an oder ersetzt es (REPLACE INTO ist
|
||||
// bei RT-Tabellen der Upsert-Mechanismus ueber die id).
|
||||
func Upsert(ctx context.Context, db *sql.DB, d Doc) error {
|
||||
const query = "REPLACE INTO " + TableName +
|
||||
" (id, document_id, title, body, tags, folder_path, document_type, created_by, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?)"
|
||||
_, err := db.ExecContext(ctx, query,
|
||||
d.ID, d.DocumentID, d.Title, d.Body, d.Tags, d.FolderPath, d.DocumentType, d.CreatedBy, d.CreatedAtSec, d.UpdatedAtSec,
|
||||
" (id, document_id, title, body, tags, folder_path, document_type, created_by, created_at, updated_at, attrs) VALUES (?,?,?,?,?,?,?,?,?,?,?)"
|
||||
attrs, err := d.Attrs.JSON()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = db.ExecContext(ctx, query,
|
||||
d.ID, d.DocumentID, d.Title, d.Body, d.Tags, d.FolderPath, d.DocumentType, d.CreatedBy, d.CreatedAtSec, d.UpdatedAtSec, attrs,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("searchindex: dokument indexieren: %w", err)
|
||||
@@ -266,6 +271,7 @@ type SearchRequest struct {
|
||||
Sort Sort
|
||||
Limit int
|
||||
Offset int
|
||||
Facet FacetFilters
|
||||
}
|
||||
|
||||
type PageHit struct {
|
||||
@@ -325,15 +331,9 @@ func BuildPageQuery(req SearchRequest) (query string, args []any, err error) {
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
where := "MATCH(?)"
|
||||
args = []any{escapeMatchTerm(req.Term)}
|
||||
for _, f := range req.Filters {
|
||||
clause, err := f.Field.clause()
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
where += " AND " + clause
|
||||
args = append(args, f.Value)
|
||||
where, args, err := buildWhere(req)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
query = "SELECT id, document_id, title, folder_path, document_type, updated_at, WEIGHT() AS score, " +
|
||||
"SNIPPET(body, QUERY(), 'limit=240', 'around=8', 'before_match=" + snippetOpen + "', 'after_match=" + snippetClose + "') AS snippet " +
|
||||
|
||||
@@ -30,7 +30,8 @@ const schemaDDL = `CREATE TABLE IF NOT EXISTS ` + TableName + ` (
|
||||
document_type string attribute,
|
||||
created_by string attribute,
|
||||
created_at timestamp,
|
||||
updated_at timestamp
|
||||
updated_at timestamp,
|
||||
attrs json
|
||||
)
|
||||
morphology='libstemmer_de'
|
||||
charset_table='non_cjk, U+00DF, U+00E4, U+00F6, U+00FC, U+00C4, U+00D6, U+00DC'
|
||||
@@ -65,6 +66,38 @@ func EnsureSchema(ctx context.Context, db *sql.DB) error {
|
||||
if _, err := db.ExecContext(ctx, schemaDDL); err != nil {
|
||||
return fmt.Errorf("searchindex: schema anlegen: %w", err)
|
||||
}
|
||||
return ensureAttrsColumn(ctx, db)
|
||||
}
|
||||
|
||||
// ensureAttrsColumn ruestet Indizes nach, die vor SRC-08 ohne attrs
|
||||
// angelegt wurden; bestehende Dokumente erhalten attrs erst per Reindex.
|
||||
func ensureAttrsColumn(ctx context.Context, db *sql.DB) error {
|
||||
rows, err := db.QueryContext(ctx, "DESCRIBE "+TableName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("searchindex: schema lesen: %w", err)
|
||||
}
|
||||
found := false
|
||||
for rows.Next() {
|
||||
var field, typ, props sql.NullString
|
||||
if err := rows.Scan(&field, &typ, &props); err != nil {
|
||||
_ = rows.Close()
|
||||
return fmt.Errorf("searchindex: schema lesen: %w", err)
|
||||
}
|
||||
if field.String == "attrs" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
err = rows.Err()
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("searchindex: schema lesen: %w", err)
|
||||
}
|
||||
if found {
|
||||
return nil
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, "ALTER TABLE "+TableName+" ADD COLUMN attrs json"); err != nil {
|
||||
return fmt.Errorf("searchindex: attrs-spalte anlegen: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -17,10 +17,10 @@ func itoa(i int) string { return strconv.Itoa(i) }
|
||||
func TestQueryBuilder_NoDynamicSQLAssembly(t *testing.T) {
|
||||
allowedConsts := map[string]bool{
|
||||
"TableName": true, "snippetOpen": true, "snippetClose": true, "MaxWindow": true,
|
||||
"where": true, "order": true, "clause": true,
|
||||
"where": true, "order": true, "clause": true, "path": true, "lo": true, "hi": true,
|
||||
}
|
||||
fset := token.NewFileSet()
|
||||
files := []string{"query.go", "schema.go"}
|
||||
files := []string{"query.go", "schema.go", "facets.go"}
|
||||
for _, name := range files {
|
||||
f, err := parser.ParseFile(fset, name, nil, 0)
|
||||
if err != nil {
|
||||
|
||||
@@ -12,7 +12,9 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
@@ -126,7 +128,12 @@ func (s *Syncer) SyncDocument(ctx context.Context, documentID string) error {
|
||||
return searchindex.Delete(ctx, s.Index, id)
|
||||
}
|
||||
|
||||
tags, err := s.loadTags(ctx, documentID)
|
||||
tagNames, err := s.loadTagNames(ctx, documentID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tags := strings.Join(tagNames, " ")
|
||||
attrs, err := s.loadAttrs(ctx, documentID, tagNames)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -151,6 +158,7 @@ func (s *Syncer) SyncDocument(ctx context.Context, documentID string) error {
|
||||
CreatedBy: createdBy,
|
||||
CreatedAtSec: createdAtSec,
|
||||
UpdatedAtSec: updatedAtSec,
|
||||
Attrs: attrs,
|
||||
}
|
||||
if err := searchindex.Upsert(ctx, s.Index, doc); err != nil {
|
||||
return err
|
||||
@@ -158,23 +166,63 @@ func (s *Syncer) SyncDocument(ctx context.Context, documentID string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Syncer) loadTags(ctx context.Context, documentID string) (string, error) {
|
||||
func (s *Syncer) loadTagNames(ctx context.Context, documentID string) ([]string, error) {
|
||||
rows, err := s.Pool.Query(ctx, `
|
||||
SELECT t.name FROM document_tags dt JOIN tags t ON t.id = dt.tag_id WHERE dt.document_id = $1 ORDER BY t.name
|
||||
`, documentID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("searchsync: tags lesen: %w", err)
|
||||
return nil, fmt.Errorf("searchsync: tags lesen: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var names []string
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
return "", fmt.Errorf("searchsync: tag-zeile lesen: %w", err)
|
||||
return nil, fmt.Errorf("searchsync: tag-zeile lesen: %w", err)
|
||||
}
|
||||
names = append(names, name)
|
||||
}
|
||||
return strings.Join(names, " "), rows.Err()
|
||||
return names, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Syncer) loadAttrs(ctx context.Context, documentID string, tagNames []string) (searchindex.Attrs, error) {
|
||||
attrs := searchindex.Attrs{Tags: tagNames, Typed: map[string]float64{}, Strings: map[string]string{}}
|
||||
rows, err := s.Pool.Query(ctx, `
|
||||
SELECT mf.field_key, mf.field_type, v.value
|
||||
FROM document_metadata_values v JOIN metadata_fields mf ON mf.id = v.field_id
|
||||
WHERE v.document_id = $1
|
||||
`, documentID)
|
||||
if err != nil {
|
||||
return attrs, fmt.Errorf("searchsync: metadatenwerte lesen: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var key, typ, value string
|
||||
if err := rows.Scan(&key, &typ, &value); err != nil {
|
||||
return attrs, fmt.Errorf("searchsync: metadatenwert lesen: %w", err)
|
||||
}
|
||||
if !searchindex.ValidCustomKey(key) {
|
||||
continue
|
||||
}
|
||||
attrs.Strings[key] = value
|
||||
switch typ {
|
||||
case "number":
|
||||
if n, err := strconv.ParseFloat(value, 64); err == nil {
|
||||
attrs.Typed[key] = n
|
||||
}
|
||||
case "date":
|
||||
if d, err := time.Parse("2006-01-02", value); err == nil {
|
||||
attrs.Typed[key] = float64(d.Unix())
|
||||
}
|
||||
case "bool":
|
||||
if b, err := strconv.ParseBool(value); err == nil && b {
|
||||
attrs.Typed[key] = 1
|
||||
} else if err == nil {
|
||||
attrs.Typed[key] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
return attrs, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Syncer) loadFolderPath(ctx context.Context, folderID *string) (string, error) {
|
||||
|
||||
Reference in new issue
Block a user