From 2a49782510546036a6071c25e8ad8ab3d5c7bcd1 Mon Sep 17 00:00:00 2001 From: patrick Date: Sun, 20 Sep 2026 01:01:20 +0200 Subject: [PATCH] SRC-08: facetten- & filter-api Co-Authored-By: Claude Sonnet 5 --- .../manticore/schema/0001_dms_documents.sql | 7 +- dms/internal/searchapi/facets_test.go | 351 ++++++++++++++++++ dms/internal/searchapi/filters.go | 168 +++++++++ dms/internal/searchapi/handler.go | 40 +- dms/internal/searchindex/custompath.go | 25 ++ dms/internal/searchindex/custompath_test.go | 67 ++++ dms/internal/searchindex/facets.go | 259 +++++++++++++ dms/internal/searchindex/query.go | 24 +- dms/internal/searchindex/schema.go | 35 +- dms/internal/searchindex/static_sql_test.go | 4 +- dms/internal/searchsync/searchsync.go | 58 ++- 11 files changed, 1013 insertions(+), 25 deletions(-) create mode 100644 dms/internal/searchapi/facets_test.go create mode 100644 dms/internal/searchapi/filters.go create mode 100644 dms/internal/searchindex/custompath.go create mode 100644 dms/internal/searchindex/custompath_test.go create mode 100644 dms/internal/searchindex/facets.go diff --git a/dms/deploy/manticore/schema/0001_dms_documents.sql b/dms/deploy/manticore/schema/0001_dms_documents.sql index 8aea42a7..fe1b5665 100644 --- a/dms/deploy/manticore/schema/0001_dms_documents.sql +++ b/dms/deploy/manticore/schema/0001_dms_documents.sql @@ -7,6 +7,10 @@ -- in internal/searchindex.BuildSearchQuery statt hier im Schema: -- title = 10, tags = 5, body = 1 -- +-- attrs (SRC-08): JSON mit tags[] (Tag-Namen), f{} (numerische/Datums- +-- Werte benutzerdefinierter Felder, Datum als Epoche) und fs{} (Rohwert je +-- Feld als String) fuer Facetten und Filter. +-- -- Deutsche Sprachbehandlung (Akzeptanzkriterium 3): -- morphology='libstemmer_de' -- Snowball-Stemmer fuer Deutsch -- charset_table erweitert um Umlaute/ß als Buchstaben (sonst als @@ -25,7 +29,8 @@ CREATE TABLE IF NOT EXISTS dms_documents ( document_type string attribute, created_by string attribute, created_at timestamp, - updated_at timestamp + updated_at timestamp, + attrs json ) morphology='libstemmer_de' charset_table='non_cjk, U+00DF, U+00E4, U+00F6, U+00FC, U+00C4, U+00D6, U+00DC' diff --git a/dms/internal/searchapi/facets_test.go b/dms/internal/searchapi/facets_test.go new file mode 100644 index 00000000..83552b5e --- /dev/null +++ b/dms/internal/searchapi/facets_test.go @@ -0,0 +1,351 @@ +package searchapi + +import ( + "context" + "fmt" + "net/url" + "os" + "sort" + "strings" + "testing" + "time" + + "gitea.perlbach24.de/scripte/nexarch/dms/internal/documenttype" + "gitea.perlbach24.de/scripte/nexarch/dms/internal/metadatafield" + "gitea.perlbach24.de/scripte/nexarch/dms/internal/searchindex" + "gitea.perlbach24.de/scripte/nexarch/dms/internal/searchsync" + "gitea.perlbach24.de/scripte/nexarch/dms/internal/tag" + "gitea.perlbach24.de/scripte/nexarch/dms/internal/testenv" +) + +func facetMap(vals []facetValueJSON) map[string]int { + m := map[string]int{} + for _, v := range vals { + m[v.Value] = v.Count + } + return m +} + +func docIDs(out responseJSON) []string { + var ids []string + for _, h := range out.Hits { + ids = append(ids, h.DocumentID) + } + sort.Strings(ids) + return ids +} + +func attrsOf(tags []string, typed map[string]float64, strs map[string]string) searchindex.Attrs { + return searchindex.Attrs{Tags: tags, Typed: typed, Strings: strs} +} + +func TestFacets_MultipleFiltersCombine(t *testing.T) { + srv, db := setup(t, denyList{}) + tok := token(t) + now := time.Now().Unix() + a := index(t, db, searchindex.Doc{Title: tok + " a", CreatedBy: "u1", DocumentType: "Rechnung", UpdatedAtSec: now, + Attrs: attrsOf([]string{"steuer", "2026"}, map[string]float64{"betrag": 100}, map[string]string{"region": "nord"})}) + index(t, db, searchindex.Doc{Title: tok + " b", CreatedBy: "u1", DocumentType: "Rechnung", UpdatedAtSec: now, + Attrs: attrsOf([]string{"steuer"}, map[string]float64{"betrag": 900}, map[string]string{"region": "nord"})}) + index(t, db, searchindex.Doc{Title: tok + " c", CreatedBy: "u1", DocumentType: "Vertrag", UpdatedAtSec: now, + Attrs: attrsOf([]string{"steuer", "2026"}, map[string]float64{"betrag": 100}, map[string]string{"region": "sued"})}) + + base := srv.URL + "/search?caller=u1&q=" + tok + cases := []struct { + name string + query string + want int + }{ + {"nur typ", "&document_type=Rechnung", 2}, + {"typ+tag", "&document_type=Rechnung&tag=2026", 1}, + {"typ+tag+feldbereich", "&document_type=Rechnung&tag=2026&field.betrag.min=50&field.betrag.max=150", 1}, + {"zwei tags", "&tag=steuer&tag=2026", 2}, + {"feld-gleichheit+typ", "&field.region=nord&document_type=Rechnung", 2}, + {"typ-oder", "&document_type=Rechnung&document_type=Vertrag", 3}, + {"widerspruch", "&document_type=Vertrag&field.region=nord", 0}, + } + for _, c := range cases { + status, out := get(t, base+c.query) + if status != 200 || out.Total != c.want || len(out.Hits) != c.want { + t.Errorf("%s: status=%d total=%d hits=%d, want %d", c.name, status, out.Total, len(out.Hits), c.want) + } + } + _, out := get(t, base+"&document_type=Rechnung&tag=2026&field.betrag.max=150") + if ids := docIDs(out); len(ids) != 1 || ids[0] != a.DocumentID { + t.Errorf("falsches dokument: %v", ids) + } +} + +func TestFacets_CountsPerValueAndUnderActiveFilters(t *testing.T) { + srv, db := setup(t, denyList{}) + tok := token(t) + now := time.Now().Unix() + for i, spec := range []struct { + typ string + tags []string + reg string + }{ + {"Rechnung", []string{"a", "b"}, "nord"}, {"Rechnung", []string{"a"}, "sued"}, {"Rechnung", nil, "nord"}, + {"Vertrag", []string{"b"}, "nord"}, {"", []string{"a"}, ""}, + } { + index(t, db, searchindex.Doc{Title: fmt.Sprintf("%s %d", tok, i), CreatedBy: "u1", DocumentType: spec.typ, UpdatedAtSec: now, + Attrs: attrsOf(spec.tags, nil, map[string]string{"region": spec.reg})}) + } + base := srv.URL + "/search?caller=u1&q=" + tok + "&facets=document_type,tag,created_by,field.region" + _, out := get(t, base) + if out.Total != 5 { + t.Fatalf("total = %d", out.Total) + } + if got := facetMap(out.Facets["document_type"]); got["Rechnung"] != 3 || got["Vertrag"] != 1 || len(got) != 2 { + t.Errorf("document_type = %v", got) + } + if got := facetMap(out.Facets["tag"]); got["a"] != 3 || got["b"] != 2 || len(got) != 2 { + t.Errorf("tag = %v", got) + } + if got := facetMap(out.Facets["created_by"]); got["u1"] != 5 || len(got) != 1 { + t.Errorf("created_by = %v", got) + } + if got := facetMap(out.Facets["field.region"]); got["nord"] != 3 || got["sued"] != 1 || len(got) != 2 { + t.Errorf("field.region = %v", got) + } + _, out = get(t, base+"&tag=a") + if got := facetMap(out.Facets["document_type"]); got["Rechnung"] != 2 || len(got) != 1 { + t.Errorf("document_type unter tag=a = %v", got) + } + if got := facetMap(out.Facets["tag"]); got["a"] != 3 || got["b"] != 1 { + t.Errorf("tag unter tag=a = %v", got) + } + if out.Facets["tag"][0].Value != "a" { + t.Errorf("facetten nicht nach anzahl absteigend: %v", out.Facets["tag"]) + } +} + +func TestFilters_DateAndValueRanges(t *testing.T) { + srv, db := setup(t, denyList{}) + tok := token(t) + day := func(s string) int64 { + d, _ := time.Parse("2006-01-02", s) + return d.Unix() + 3600 + } + for i, spec := range []struct { + updated string + betrag float64 + frist string + }{ + {"2026-01-10", 10, "2026-03-01"}, {"2026-02-10", 20, "2026-04-01"}, {"2026-03-10", 30, "2026-05-01"}, + } { + frist, _ := time.Parse("2006-01-02", spec.frist) + index(t, db, searchindex.Doc{Title: fmt.Sprintf("%s %d", tok, i), CreatedBy: "u1", UpdatedAtSec: day(spec.updated), + Attrs: attrsOf(nil, map[string]float64{"betrag": spec.betrag, "frist": float64(frist.Unix())}, nil)}) + } + base := srv.URL + "/search?caller=u1&q=" + tok + for _, c := range []struct { + query string + want int + }{ + {"&updated_from=2026-02-01", 2}, + {"&updated_to=2026-02-10", 2}, + {"&updated_from=2026-02-10&updated_to=2026-02-10", 1}, + {"&updated_from=2026-01-15&updated_to=2026-03-01", 1}, + {"&created_from=2026-01-01T00:00:00Z", 3}, + {"&field.betrag.min=15&field.betrag.max=25", 1}, + {"&field.betrag.min=20", 2}, + {"&field.frist.min=2026-04-01&field.frist.max=2026-04-30", 1}, + {"&field.frist.max=2026-03-31", 1}, + } { + if status, out := get(t, base+c.query); status != 200 || out.Total != c.want { + t.Errorf("%s: status=%d total=%d, want %d", c.query, status, out.Total, c.want) + } + } +} + +func TestFilters_InvalidParametersRejected(t *testing.T) { + srv, _ := setup(t, denyList{}) + base := srv.URL + "/search?caller=u1&q=x" + for _, q := range []string{ + "&updated_from=gestern", "&created_from=2026-03-01&created_to=2026-01-01", "&field.Betrag=x", "&field.a.b.min=1", + "&field.betrag.min=abc", "&facets=passwort", "&facets=field.a;drop", "&field.x%3Bdrop=1", "&field.betrag.avg=1", + } { + if status, _ := get(t, base+q); status != 400 { + t.Errorf("%s: status = %d, want 400", q, status) + } + } +} + +// Pruefung 2: leeres Ergebnis bleibt stabil (Struktur, Facetten leer, kein Fehler). +func TestFilters_EmptyResultIsStable(t *testing.T) { + srv, db := setup(t, denyList{}) + tok := token(t) + index(t, db, searchindex.Doc{Title: tok, CreatedBy: "u1", DocumentType: "Rechnung"}) + status, out := get(t, srv.URL+"/search?caller=u1&q="+tok+"&document_type=Nichts&tag=x&field.a.min=1&facets=document_type,tag,field.a") + if status != 200 || out.Total != 0 || out.Hits == nil || len(out.Hits) != 0 { + t.Fatalf("status=%d total=%d hits=%v", status, out.Total, out.Hits) + } + for _, name := range []string{"document_type", "tag", "field.a"} { + vals, ok := out.Facets[name] + if !ok || len(vals) != 0 { + t.Errorf("facette %s = %v (vorhanden=%v), want leer", name, vals, ok) + } + } +} + +// Pruefung 3: Antwortzeit mit fuenf aktiven Filtern. +func TestFilters_ResponseTimeWithFiveFilters(t *testing.T) { + srv, db := setup(t, denyList{}) + tok := token(t) + now := time.Now().Unix() + for i := 0; i < 200; i++ { + index(t, db, searchindex.Doc{Title: fmt.Sprintf("%s %d", tok, i), CreatedBy: "u1", DocumentType: "Rechnung", UpdatedAtSec: now, + Attrs: attrsOf([]string{"steuer", "2026"}, map[string]float64{"betrag": float64(i)}, map[string]string{"region": "nord"})}) + } + q := url.Values{"caller": {"u1"}, "q": {tok}, "document_type": {"Rechnung"}, "tag": {"steuer", "2026"}, + "updated_from": {"2020-01-01"}, "field.betrag.min": {"10"}, "field.region": {"nord"}, "facets": {"document_type,tag,field.region"}} + var durations []time.Duration + for i := 0; i < 30; i++ { + start := time.Now() + status, out := get(t, srv.URL+"/search?"+q.Encode()) + durations = append(durations, time.Since(start)) + if status != 200 || out.Total != 190 { + t.Fatalf("status=%d total=%d", status, out.Total) + } + } + sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) + p95 := durations[len(durations)*95/100] + t.Logf("5 filter + 3 facetten: median=%v p95=%v max=%v", durations[len(durations)/2], p95, durations[len(durations)-1]) + if p95 > responseTimeLimit { + t.Errorf("p95 = %v ueber %v", p95, responseTimeLimit) + } +} + +// Pruefung 1: Facetten- und Trefferzahlen gegen SQL auf der Quelldatenbank abgeglichen. +func TestFacets_CountsMatchDatabaseQuery(t *testing.T) { + if os.Getenv("TEST_TENANT_DSN") == "" { + t.Skip("TEST_TENANT_DSN nicht gesetzt") + } + srv, idx := setup(t, denyList{}) + e := testenv.New(t) + ctx := context.Background() + syncer := &searchsync.Syncer{Pool: e.Pool, Index: idx} + tok := token(t) + + types := map[string]string{} + for _, n := range []string{"Rechnung", "Vertrag"} { + dt, err := documenttype.Create(ctx, e.Pool, n, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + types[n] = dt.ID + } + tags := map[string]string{} + for _, n := range []string{"steuer", "2026", "intern"} { + tg, err := tag.Create(ctx, e.Pool, n, "", nil) + if err != nil { + t.Fatal(err) + } + tags[n] = tg.ID + } + fields := map[string]string{} + for key, typ := range map[string]string{"betrag": "number", "region": "select", "frist": "date"} { + f, err := metadatafield.Create(ctx, e.Pool, key, key, typ, false, nil, nil) + if err != nil { + t.Fatalf("feld %s: %v", key, err) + } + fields[key] = f.ID + } + + typeNames := []string{"Rechnung", "Vertrag"} + tagNames := []string{"steuer", "2026", "intern"} + regions := []string{"nord", "sued", "west"} + const n = 36 + for i := 0; i < n; i++ { + id, _ := e.UploadDoc(t, fmt.Sprintf("%s-%02d.txt", tok, i), "text/plain", []byte(fmt.Sprintf("inhalt %s %d", tok, i))) + t.Cleanup(func() { _ = searchindex.Delete(ctx, idx, searchsync.DocIndexID(id)) }) + if _, err := documenttype.ChangeDocumentType(ctx, e.Pool, id, types[typeNames[i%2]], false); err != nil { + t.Fatal(err) + } + for j, tn := range tagNames { + if (i>>j)&1 == 1 || (i%5 == 0 && j == 0) { + if err := tag.Assign(ctx, e.Pool, id, tags[tn]); err != nil { + t.Fatal(err) + } + } + } + for key, val := range map[string]string{ + "betrag": fmt.Sprintf("%d", (i*37)%500), "region": regions[i%3], + "frist": time.Date(2026, time.Month(1+i%6), 1+i%27, 0, 0, 0, 0, time.UTC).Format("2006-01-02"), + } { + if err := metadatafield.SetValue(ctx, e.Pool, id, fields[key], val); err != nil { + t.Fatalf("wert %s=%s: %v", key, val, err) + } + } + if err := syncer.SyncDocument(ctx, id); err != nil { + t.Fatalf("sync: %v", err) + } + } + + count := func(sqlText string, args ...any) int { + t.Helper() + var c int + if err := e.Pool.QueryRow(ctx, sqlText, args...).Scan(&c); err != nil { + t.Fatalf("sql: %v", err) + } + return c + } + facetsQ := "&facets=document_type,tag,field.region" + filterQ := "&document_type=Rechnung&tag=steuer&field.betrag.min=100&field.betrag.max=400&field.frist.min=2026-02-01&field.frist.max=2026-05-31" + const filterSQL = ` + FROM documents d + JOIN document_types dt ON dt.id = d.document_type_id + WHERE d.title LIKE $1 AND dt.name = 'Rechnung' + AND EXISTS (SELECT 1 FROM document_tags x JOIN tags t ON t.id = x.tag_id WHERE x.document_id = d.id AND t.name = 'steuer') + AND EXISTS (SELECT 1 FROM document_metadata_values v JOIN metadata_fields f ON f.id = v.field_id + WHERE v.document_id = d.id AND f.field_key = 'betrag' AND v.value::numeric BETWEEN 100 AND 400) + AND EXISTS (SELECT 1 FROM document_metadata_values v JOIN metadata_fields f ON f.id = v.field_id + WHERE v.document_id = d.id AND f.field_key = 'frist' AND v.value::date BETWEEN '2026-02-01' AND '2026-05-31')` + like := tok + "%" + + base := srv.URL + "/search?caller=" + e.UserID + "&limit=100&q=" + tok + _, all := get(t, base+facetsQ) + if want := count(`SELECT count(*) FROM documents WHERE title LIKE $1`, like); all.Total != want || want != n { + t.Fatalf("gesamt: index=%d db=%d, erwartet %d", all.Total, want, n) + } + for _, name := range typeNames { + want := count(`SELECT count(*) FROM documents d JOIN document_types dt ON dt.id = d.document_type_id WHERE d.title LIKE $1 AND dt.name = $2`, like, name) + if got := facetMap(all.Facets["document_type"])[name]; got != want { + t.Errorf("facette typ %s: index=%d db=%d", name, got, want) + } + } + for _, name := range tagNames { + want := count(`SELECT count(DISTINCT d.id) FROM documents d JOIN document_tags x ON x.document_id = d.id JOIN tags t ON t.id = x.tag_id WHERE d.title LIKE $1 AND t.name = $2`, like, name) + if got := facetMap(all.Facets["tag"])[name]; got != want { + t.Errorf("facette tag %s: index=%d db=%d", name, got, want) + } + } + for _, r := range regions { + want := count(`SELECT count(*) FROM documents d JOIN document_metadata_values v ON v.document_id = d.id JOIN metadata_fields f ON f.id = v.field_id WHERE d.title LIKE $1 AND f.field_key = 'region' AND v.value = $2`, like, r) + if got := facetMap(all.Facets["field.region"])[r]; got != want { + t.Errorf("facette region %s: index=%d db=%d", r, got, want) + } + } + + _, filtered := get(t, base+filterQ+facetsQ) + wantTotal := count(`SELECT count(*) `+filterSQL, like) + if wantTotal == 0 { + t.Fatal("testdaten ergeben leere filtermenge, pruefung sinnlos") + } + if filtered.Total != wantTotal || len(filtered.Hits) != wantTotal { + t.Errorf("gefilterte treffer: index=%d/%d db=%d", filtered.Total, len(filtered.Hits), wantTotal) + } + for _, r := range regions { + want := count(`SELECT count(*) `+filterSQL+` AND EXISTS (SELECT 1 FROM document_metadata_values v JOIN metadata_fields f ON f.id = v.field_id WHERE v.document_id = d.id AND f.field_key = 'region' AND v.value = $2)`, like, r) + if got := facetMap(filtered.Facets["field.region"])[r]; got != want { + t.Errorf("gefilterte facette region %s: index=%d db=%d", r, got, want) + } + } + if got := facetMap(filtered.Facets["document_type"]); len(got) != 1 || got["Rechnung"] != wantTotal { + t.Errorf("gefilterte facette typ = %v, want Rechnung=%d", got, wantTotal) + } + if strings.TrimSpace(tok) == "" { + t.Fatal("token leer") + } +} diff --git a/dms/internal/searchapi/filters.go b/dms/internal/searchapi/filters.go new file mode 100644 index 00000000..37dec3c8 --- /dev/null +++ b/dms/internal/searchapi/filters.go @@ -0,0 +1,168 @@ +package searchapi + +import ( + "fmt" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "gitea.perlbach24.de/scripte/nexarch/dms/internal/searchindex" +) + +const fieldPrefix = "field." + +func invalid(format string, a ...any) error { + return fmt.Errorf("%w: %s", searchindex.ErrInvalidRequest, fmt.Sprintf(format, a...)) +} + +func facetName(s searchindex.FacetSpec) string { + switch s.Kind { + case searchindex.FacetDocumentType: + return "document_type" + case searchindex.FacetTag: + return "tag" + case searchindex.FacetCreatedBy: + return "created_by" + default: + return fieldPrefix + s.Key + } +} + +// parseTime akzeptiert YYYY-MM-DD oder RFC3339; endOfDay macht ein reines +// Datum zur letzten Sekunde des Tages (obere Bereichsgrenze). +func parseTime(v string, endOfDay bool) (int64, error) { + if d, err := time.Parse("2006-01-02", v); err == nil { + if endOfDay { + d = d.Add(24*time.Hour - time.Second) + } + return d.Unix(), nil + } + t, err := time.Parse(time.RFC3339, v) + if err != nil { + return 0, invalid("zeitwert %q ungueltig (YYYY-MM-DD oder RFC3339)", v) + } + return t.Unix(), nil +} + +func parseBound(v string, endOfDay bool) (float64, error) { + if n, err := strconv.ParseFloat(v, 64); err == nil { + return n, nil + } + t, err := parseTime(v, endOfDay) + if err != nil { + return 0, invalid("bereichswert %q ist weder zahl noch datum", v) + } + return float64(t), nil +} + +func parseFacetParams(q url.Values) (searchindex.FacetFilters, []searchindex.FacetSpec, error) { + var ff searchindex.FacetFilters + var specs []searchindex.FacetSpec + ff.DocumentTypes = nonEmpty(q["document_type"]) + ff.Tags = nonEmpty(q["tag"]) + + for _, r := range []struct { + field searchindex.RangeField + from, to string + }{ + {searchindex.RangeCreatedAt, "created_from", "created_to"}, + {searchindex.RangeUpdatedAt, "updated_from", "updated_to"}, + } { + rg := searchindex.Range{Field: r.field} + if v := q.Get(r.from); v != "" { + t, err := parseTime(v, false) + if err != nil { + return ff, nil, err + } + rg.Min = &t + } + if v := q.Get(r.to); v != "" { + t, err := parseTime(v, true) + if err != nil { + return ff, nil, err + } + rg.Max = &t + } + if rg.Min != nil && rg.Max != nil && *rg.Min > *rg.Max { + return ff, nil, invalid("%s liegt nach %s", r.from, r.to) + } + if rg.Min != nil || rg.Max != nil { + ff.Ranges = append(ff.Ranges, rg) + } + } + + custom := map[string]*searchindex.CustomFilter{} + var keys []string + for name, vals := range q { + if !strings.HasPrefix(name, fieldPrefix) || len(vals) == 0 || vals[0] == "" { + continue + } + key, part := strings.TrimPrefix(name, fieldPrefix), "" + if i := strings.LastIndex(key, "."); i >= 0 { + key, part = key[:i], key[i+1:] + if part != "min" && part != "max" { + return ff, nil, invalid("parameter %q unbekannt", name) + } + } + if !searchindex.ValidCustomKey(key) { + return ff, nil, invalid("feldschluessel %q nicht erlaubt", key) + } + c := custom[key] + if c == nil { + c = &searchindex.CustomFilter{Key: key} + custom[key] = c + keys = append(keys, key) + } + switch part { + case "": + v := vals[0] + c.Eq = &v + case "min": + n, err := parseBound(vals[0], false) + if err != nil { + return ff, nil, err + } + c.Min = &n + case "max": + n, err := parseBound(vals[0], true) + if err != nil { + return ff, nil, err + } + c.Max = &n + } + } + sort.Strings(keys) + for _, k := range keys { + ff.Custom = append(ff.Custom, *custom[k]) + } + + if v := strings.TrimSpace(q.Get("facets")); v != "" { + for _, name := range strings.Split(v, ",") { + switch name = strings.TrimSpace(name); { + case name == "document_type": + specs = append(specs, searchindex.FacetSpec{Kind: searchindex.FacetDocumentType}) + case name == "tag": + specs = append(specs, searchindex.FacetSpec{Kind: searchindex.FacetTag}) + case name == "created_by": + specs = append(specs, searchindex.FacetSpec{Kind: searchindex.FacetCreatedBy}) + case strings.HasPrefix(name, fieldPrefix) && searchindex.ValidCustomKey(strings.TrimPrefix(name, fieldPrefix)): + specs = append(specs, searchindex.FacetSpec{Kind: searchindex.FacetCustom, Key: strings.TrimPrefix(name, fieldPrefix)}) + default: + return ff, nil, invalid("facette %q nicht erlaubt", name) + } + } + } + return ff, specs, nil +} + +func nonEmpty(in []string) []string { + var out []string + for _, v := range in { + if v != "" { + out = append(out, v) + } + } + return out +} diff --git a/dms/internal/searchapi/handler.go b/dms/internal/searchapi/handler.go index 7abc6327..9de1ebfd 100644 --- a/dms/internal/searchapi/handler.go +++ b/dms/internal/searchapi/handler.go @@ -48,6 +48,13 @@ type responseJSON struct { Offset int `json:"offset"` TookMS int64 `json:"took_ms"` Hits []hitJSON `json:"hits"` + // Facets nur bei gesetztem Parameter "facets"; Schluessel wie dort angegeben. + Facets map[string][]facetValueJSON `json:"facets,omitempty"` +} + +type facetValueJSON struct { + Value string `json:"value"` + Count int `json:"count"` } func writeError(w http.ResponseWriter, status int, message string) { @@ -105,7 +112,7 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { return } filters := []searchindex.Filter{{Field: searchindex.FilterCreatedBy, Value: caller}} - for _, name := range []string{"document_type", "folder_path"} { + for _, name := range []string{"folder_path", "created_by"} { if v := q.Get(name); v != "" { field, err := searchindex.ParseFilterField(name) if err != nil { @@ -115,10 +122,16 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { filters = append(filters, searchindex.Filter{Field: field, Value: v}) } } + facetFilters, specs, err := parseFacetParams(q) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + searchReq := searchindex.SearchRequest{ + Term: term, Filters: filters, Sort: sortVal, Limit: limit, Offset: offset, Facet: facetFilters, + } - page, err := searchindex.SearchPage(r.Context(), s.index, searchindex.SearchRequest{ - Term: term, Filters: filters, Sort: sortVal, Limit: limit, Offset: offset, - }) + page, err := searchindex.SearchPage(r.Context(), s.index, searchReq) if err != nil { if errors.Is(err, searchindex.ErrInvalidRequest) { writeError(w, http.StatusBadRequest, err.Error()) @@ -146,6 +159,25 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { Score: h.Score, UpdatedAt: h.UpdatedAt, SnippetHTML: renderSnippet(h.Snippet), }) } + if len(specs) > 0 { + results, err := searchindex.Facets(r.Context(), s.index, searchReq, specs) + if err != nil { + if errors.Is(err, searchindex.ErrInvalidRequest) { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + resp.Facets = map[string][]facetValueJSON{} + for _, fr := range results { + vals := make([]facetValueJSON, 0, len(fr.Values)) + for _, v := range fr.Values { + vals = append(vals, facetValueJSON{Value: v.Value, Count: v.Count}) + } + resp.Facets[facetName(fr.Spec)] = vals + } + } resp.TookMS = time.Since(start).Milliseconds() w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(resp) diff --git a/dms/internal/searchindex/custompath.go b/dms/internal/searchindex/custompath.go new file mode 100644 index 00000000..0d0f8a91 --- /dev/null +++ b/dms/internal/searchindex/custompath.go @@ -0,0 +1,25 @@ +package searchindex + +import ( + "fmt" + "regexp" +) + +// Feldschluessel benutzerdefinierter Felder duerfen als JSON-Pfad in die +// Abfrage; deshalb nur dieses enge Alphabet. Diese Datei ist die einzige +// Stelle, an der ein Nutzerwert Teil des SQL-Textes wird. +var customKeyPattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,62}$`) + +func ValidCustomKey(key string) bool { return customKeyPattern.MatchString(key) } + +const ( + pathTyped = "attrs.f." + pathString = "attrs.fs." +) + +func customPath(prefix, key string) (string, error) { + if !ValidCustomKey(key) { + return "", fmt.Errorf("%w: feldschluessel %q nicht erlaubt", ErrInvalidRequest, key) + } + return prefix + key, nil +} diff --git a/dms/internal/searchindex/custompath_test.go b/dms/internal/searchindex/custompath_test.go new file mode 100644 index 00000000..6e14c683 --- /dev/null +++ b/dms/internal/searchindex/custompath_test.go @@ -0,0 +1,67 @@ +package searchindex + +import ( + "errors" + "strings" + "testing" +) + +func TestCustomPath_RejectsAnythingButPlainKeys(t *testing.T) { + for _, key := range []string{"", "Betrag", "1abc", "a b", "a.b", "a;drop", "a'b", "a)or(1=1", "a-b", strings.Repeat("a", 64), "ä"} { + if _, err := customPath(pathString, key); !errors.Is(err, ErrInvalidRequest) { + t.Errorf("key %q: err = %v, want ErrInvalidRequest", key, err) + } + } + got, err := customPath(pathTyped, "betrag_netto") + if err != nil || got != "attrs.f.betrag_netto" { + t.Errorf("got %q, %v", got, err) + } +} + +func TestBuildPageQuery_CombinedFilters(t *testing.T) { + min, max := int64(10), int64(20) + lo, hi, eq := 1.5, 9.0, "nord" + q, args, err := BuildPageQuery(SearchRequest{ + Term: "x", + Filters: []Filter{{Field: FilterCreatedBy, Value: "u1"}}, + Facet: FacetFilters{ + DocumentTypes: []string{"a", "b"}, + Tags: []string{"t1", "t2"}, + Ranges: []Range{{Field: RangeUpdatedAt, Min: &min, Max: &max}}, + Custom: []CustomFilter{{Key: "betrag", Min: &lo, Max: &hi}, {Key: "region", Eq: &eq}}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "created_by = ?", "document_type IN (?, ?)", "attrs.tags = ? AND attrs.tags = ?", + "updated_at >= ? AND updated_at <= ?", "attrs.f.betrag >= ? AND attrs.f.betrag <= ?", "attrs.fs.region = ?", + } { + if !strings.Contains(q, want) { + t.Errorf("query enthaelt %q nicht: %s", want, q) + } + } + if len(args) != 11 { + t.Errorf("args = %d, want 11 (%v)", len(args), args) + } + for _, a := range args { + if s, ok := a.(string); ok && strings.Contains(q, s) && len(s) > 3 { + t.Errorf("nutzerwert %q steht im sql-text", s) + } + } +} + +func TestBuildFacetQuery_RejectsBadInput(t *testing.T) { + req := SearchRequest{Term: "x"} + if _, _, err := BuildFacetQuery(req, nil); !errors.Is(err, ErrInvalidRequest) { + t.Errorf("keine facetten: %v", err) + } + if _, _, err := BuildFacetQuery(req, []FacetSpec{{Kind: FacetCustom, Key: "x; drop"}}); !errors.Is(err, ErrInvalidRequest) { + t.Errorf("boeser key: %v", err) + } + many := make([]string, MaxFilterEntries+1) + if _, _, err := BuildFacetQuery(SearchRequest{Term: "x", Facet: FacetFilters{Tags: many}}, []FacetSpec{{Kind: FacetTag}}); !errors.Is(err, ErrInvalidRequest) { + t.Errorf("zu viele tags: %v", err) + } +} diff --git a/dms/internal/searchindex/facets.go b/dms/internal/searchindex/facets.go new file mode 100644 index 00000000..efda7515 --- /dev/null +++ b/dms/internal/searchindex/facets.go @@ -0,0 +1,259 @@ +package searchindex + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strconv" +) + +const ( + MaxFacetValues = 50 + MaxFilterEntries = 20 +) + +// Attrs ist der Inhalt der JSON-Spalte attrs: Tag-Namen, numerische bzw. +// Datumswerte (Epoche) und Rohwerte benutzerdefinierter Felder. +type Attrs struct { + Tags []string `json:"tags"` + Typed map[string]float64 `json:"f"` + Strings map[string]string `json:"fs"` +} + +func (a Attrs) JSON() (string, error) { + if a.Tags == nil { + a.Tags = []string{} + } + if a.Typed == nil { + a.Typed = map[string]float64{} + } + if a.Strings == nil { + a.Strings = map[string]string{} + } + b, err := json.Marshal(a) + if err != nil { + return "", fmt.Errorf("searchindex: attrs serialisieren: %w", err) + } + return string(b), nil +} + +type RangeField int + +const ( + RangeCreatedAt RangeField = iota + RangeUpdatedAt +) + +// Range ist ein geschlossenes Zeitintervall in Epochensekunden; nil = offen. +type Range struct { + Field RangeField + Min *int64 + Max *int64 +} + +// CustomFilter filtert ein benutzerdefiniertes Feld: Eq gegen den Rohwert, +// Min/Max gegen den numerischen bzw. Datumswert. +type CustomFilter struct { + Key string + Eq *string + Min *float64 + Max *float64 +} + +type FacetKind int + +const ( + FacetDocumentType FacetKind = iota + FacetTag + FacetCreatedBy + FacetCustom +) + +type FacetSpec struct { + Kind FacetKind + Key string +} + +// FacetFilters sind die kombinierbaren Filter jenseits der Einzel-Gleichheit: +// DocumentTypes = ODER, Tags/Ranges/Custom = UND. +type FacetFilters struct { + DocumentTypes []string + Tags []string + Ranges []Range + Custom []CustomFilter +} + +type FacetValue struct { + Value string + Count int +} + +type FacetResult struct { + Spec FacetSpec + Values []FacetValue +} + +func buildWhere(req SearchRequest) (where string, args []any, err error) { + where = "MATCH(?)" + args = []any{escapeMatchTerm(req.Term)} + for _, f := range req.Filters { + clause, err := f.Field.clause() + if err != nil { + return "", nil, err + } + where += " AND " + clause + args = append(args, f.Value) + } + fl := req.Facet + if len(fl.DocumentTypes) > MaxFilterEntries || len(fl.Tags) > MaxFilterEntries || + len(fl.Ranges) > MaxFilterEntries || len(fl.Custom) > MaxFilterEntries { + return "", nil, fmt.Errorf("%w: hoechstens %d werte je filterart", ErrInvalidRequest, MaxFilterEntries) + } + if len(fl.DocumentTypes) > 0 { + where += " AND document_type IN (" + for i, v := range fl.DocumentTypes { + if i > 0 { + where += ", " + } + where += "?" + args = append(args, v) + } + where += ")" + } + for _, tag := range fl.Tags { + where += " AND attrs.tags = ?" + args = append(args, tag) + } + for _, r := range fl.Ranges { + lo, hi, err := r.Field.clauses() + if err != nil { + return "", nil, err + } + if r.Min != nil { + where += " AND " + lo + args = append(args, *r.Min) + } + if r.Max != nil { + where += " AND " + hi + args = append(args, *r.Max) + } + } + for _, c := range fl.Custom { + if c.Eq != nil { + path, err := customPath(pathString, c.Key) + if err != nil { + return "", nil, err + } + where += " AND " + path + " = ?" + args = append(args, *c.Eq) + } + if c.Min != nil || c.Max != nil { + path, err := customPath(pathTyped, c.Key) + if err != nil { + return "", nil, err + } + if c.Min != nil { + where += " AND " + path + " >= ?" + args = append(args, *c.Min) + } + if c.Max != nil { + where += " AND " + path + " <= ?" + args = append(args, *c.Max) + } + } + } + return where, args, nil +} + +func (f RangeField) clauses() (lo, hi string, err error) { + switch f { + case RangeCreatedAt: + return "created_at >= ?", "created_at <= ?", nil + case RangeUpdatedAt: + return "updated_at >= ?", "updated_at <= ?", nil + default: + return "", "", fmt.Errorf("%w: unbekanntes bereichsfeld %d", ErrInvalidRequest, f) + } +} + +func (s FacetSpec) clause() (string, error) { + switch s.Kind { + case FacetDocumentType: + return " FACET document_type ORDER BY COUNT(*) DESC LIMIT " + strconv.Itoa(MaxFacetValues), nil + case FacetTag: + return " FACET attrs.tags ORDER BY COUNT(*) DESC LIMIT " + strconv.Itoa(MaxFacetValues), nil + case FacetCreatedBy: + return " FACET created_by ORDER BY COUNT(*) DESC LIMIT " + strconv.Itoa(MaxFacetValues), nil + case FacetCustom: + path, err := customPath(pathString, s.Key) + if err != nil { + return "", err + } + return " FACET " + path + " ORDER BY COUNT(*) DESC LIMIT " + strconv.Itoa(MaxFacetValues), nil + default: + return "", fmt.Errorf("%w: unbekannte facette %d", ErrInvalidRequest, s.Kind) + } +} + +func BuildFacetQuery(req SearchRequest, specs []FacetSpec) (query string, args []any, err error) { + if len(specs) == 0 || len(specs) > MaxFilterEntries { + return "", nil, fmt.Errorf("%w: 1 bis %d facetten erforderlich", ErrInvalidRequest, MaxFilterEntries) + } + where, args, err := buildWhere(req) + if err != nil { + return "", nil, err + } + query = "SELECT id FROM " + TableName + " WHERE " + where + " LIMIT 1" + for _, s := range specs { + clause, err := s.clause() + if err != nil { + return "", nil, err + } + query += clause + } + return query, args, nil +} + +// Facets liefert je Spezifikation die Trefferzahlen je Wert unter allen +// aktiven Filtern; leere/fehlende Werte werden ausgelassen. +func Facets(ctx context.Context, db *sql.DB, req SearchRequest, specs []FacetSpec) ([]FacetResult, error) { + query, args, err := BuildFacetQuery(req, specs) + if err != nil { + return nil, err + } + conn, err := db.Conn(ctx) + if err != nil { + return nil, fmt.Errorf("searchindex: verbindung: %w", err) + } + defer conn.Close() + rows, err := conn.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("searchindex: facetten: %w", err) + } + defer rows.Close() + for rows.Next() { + } + out := make([]FacetResult, 0, len(specs)) + for _, spec := range specs { + if !rows.NextResultSet() { + return nil, fmt.Errorf("searchindex: facettenergebnis fehlt: %w", rows.Err()) + } + res := FacetResult{Spec: spec, Values: []FacetValue{}} + for rows.Next() { + var v sql.NullString + var n int + if err := rows.Scan(&v, &n); err != nil { + return nil, fmt.Errorf("searchindex: facette lesen: %w", err) + } + if v.Valid && v.String != "" { + res.Values = append(res.Values, FacetValue{Value: v.String, Count: n}) + } + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("searchindex: facette lesen: %w", err) + } + out = append(out, res) + } + return out, nil +} diff --git a/dms/internal/searchindex/query.go b/dms/internal/searchindex/query.go index 0398ab7a..92415734 100644 --- a/dms/internal/searchindex/query.go +++ b/dms/internal/searchindex/query.go @@ -199,15 +199,20 @@ type Doc struct { CreatedBy string CreatedAtSec int64 UpdatedAtSec int64 + Attrs Attrs } // Upsert legt ein Dokument im Index an oder ersetzt es (REPLACE INTO ist // bei RT-Tabellen der Upsert-Mechanismus ueber die id). func Upsert(ctx context.Context, db *sql.DB, d Doc) error { const query = "REPLACE INTO " + TableName + - " (id, document_id, title, body, tags, folder_path, document_type, created_by, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?)" - _, err := db.ExecContext(ctx, query, - d.ID, d.DocumentID, d.Title, d.Body, d.Tags, d.FolderPath, d.DocumentType, d.CreatedBy, d.CreatedAtSec, d.UpdatedAtSec, + " (id, document_id, title, body, tags, folder_path, document_type, created_by, created_at, updated_at, attrs) VALUES (?,?,?,?,?,?,?,?,?,?,?)" + attrs, err := d.Attrs.JSON() + if err != nil { + return err + } + _, err = db.ExecContext(ctx, query, + d.ID, d.DocumentID, d.Title, d.Body, d.Tags, d.FolderPath, d.DocumentType, d.CreatedBy, d.CreatedAtSec, d.UpdatedAtSec, attrs, ) if err != nil { return fmt.Errorf("searchindex: dokument indexieren: %w", err) @@ -266,6 +271,7 @@ type SearchRequest struct { Sort Sort Limit int Offset int + Facet FacetFilters } type PageHit struct { @@ -325,15 +331,9 @@ func BuildPageQuery(req SearchRequest) (query string, args []any, err error) { if err != nil { return "", nil, err } - where := "MATCH(?)" - args = []any{escapeMatchTerm(req.Term)} - for _, f := range req.Filters { - clause, err := f.Field.clause() - if err != nil { - return "", nil, err - } - where += " AND " + clause - args = append(args, f.Value) + where, args, err := buildWhere(req) + if err != nil { + return "", nil, err } query = "SELECT id, document_id, title, folder_path, document_type, updated_at, WEIGHT() AS score, " + "SNIPPET(body, QUERY(), 'limit=240', 'around=8', 'before_match=" + snippetOpen + "', 'after_match=" + snippetClose + "') AS snippet " + diff --git a/dms/internal/searchindex/schema.go b/dms/internal/searchindex/schema.go index 97f25bf0..097a17e5 100644 --- a/dms/internal/searchindex/schema.go +++ b/dms/internal/searchindex/schema.go @@ -30,7 +30,8 @@ const schemaDDL = `CREATE TABLE IF NOT EXISTS ` + TableName + ` ( document_type string attribute, created_by string attribute, created_at timestamp, - updated_at timestamp + updated_at timestamp, + attrs json ) morphology='libstemmer_de' charset_table='non_cjk, U+00DF, U+00E4, U+00F6, U+00FC, U+00C4, U+00D6, U+00DC' @@ -65,6 +66,38 @@ func EnsureSchema(ctx context.Context, db *sql.DB) error { if _, err := db.ExecContext(ctx, schemaDDL); err != nil { return fmt.Errorf("searchindex: schema anlegen: %w", err) } + return ensureAttrsColumn(ctx, db) +} + +// ensureAttrsColumn ruestet Indizes nach, die vor SRC-08 ohne attrs +// angelegt wurden; bestehende Dokumente erhalten attrs erst per Reindex. +func ensureAttrsColumn(ctx context.Context, db *sql.DB) error { + rows, err := db.QueryContext(ctx, "DESCRIBE "+TableName) + if err != nil { + return fmt.Errorf("searchindex: schema lesen: %w", err) + } + found := false + for rows.Next() { + var field, typ, props sql.NullString + if err := rows.Scan(&field, &typ, &props); err != nil { + _ = rows.Close() + return fmt.Errorf("searchindex: schema lesen: %w", err) + } + if field.String == "attrs" { + found = true + } + } + err = rows.Err() + _ = rows.Close() + if err != nil { + return fmt.Errorf("searchindex: schema lesen: %w", err) + } + if found { + return nil + } + if _, err := db.ExecContext(ctx, "ALTER TABLE "+TableName+" ADD COLUMN attrs json"); err != nil { + return fmt.Errorf("searchindex: attrs-spalte anlegen: %w", err) + } return nil } diff --git a/dms/internal/searchindex/static_sql_test.go b/dms/internal/searchindex/static_sql_test.go index 524c55ec..5122ca7b 100644 --- a/dms/internal/searchindex/static_sql_test.go +++ b/dms/internal/searchindex/static_sql_test.go @@ -17,10 +17,10 @@ func itoa(i int) string { return strconv.Itoa(i) } func TestQueryBuilder_NoDynamicSQLAssembly(t *testing.T) { allowedConsts := map[string]bool{ "TableName": true, "snippetOpen": true, "snippetClose": true, "MaxWindow": true, - "where": true, "order": true, "clause": true, + "where": true, "order": true, "clause": true, "path": true, "lo": true, "hi": true, } fset := token.NewFileSet() - files := []string{"query.go", "schema.go"} + files := []string{"query.go", "schema.go", "facets.go"} for _, name := range files { f, err := parser.ParseFile(fset, name, nil, 0) if err != nil { diff --git a/dms/internal/searchsync/searchsync.go b/dms/internal/searchsync/searchsync.go index 6dbc3c88..be14e1c4 100644 --- a/dms/internal/searchsync/searchsync.go +++ b/dms/internal/searchsync/searchsync.go @@ -12,7 +12,9 @@ import ( "errors" "fmt" "hash/fnv" + "strconv" "strings" + "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -126,7 +128,12 @@ func (s *Syncer) SyncDocument(ctx context.Context, documentID string) error { return searchindex.Delete(ctx, s.Index, id) } - tags, err := s.loadTags(ctx, documentID) + tagNames, err := s.loadTagNames(ctx, documentID) + if err != nil { + return err + } + tags := strings.Join(tagNames, " ") + attrs, err := s.loadAttrs(ctx, documentID, tagNames) if err != nil { return err } @@ -151,6 +158,7 @@ func (s *Syncer) SyncDocument(ctx context.Context, documentID string) error { CreatedBy: createdBy, CreatedAtSec: createdAtSec, UpdatedAtSec: updatedAtSec, + Attrs: attrs, } if err := searchindex.Upsert(ctx, s.Index, doc); err != nil { return err @@ -158,23 +166,63 @@ func (s *Syncer) SyncDocument(ctx context.Context, documentID string) error { return nil } -func (s *Syncer) loadTags(ctx context.Context, documentID string) (string, error) { +func (s *Syncer) loadTagNames(ctx context.Context, documentID string) ([]string, error) { rows, err := s.Pool.Query(ctx, ` SELECT t.name FROM document_tags dt JOIN tags t ON t.id = dt.tag_id WHERE dt.document_id = $1 ORDER BY t.name `, documentID) if err != nil { - return "", fmt.Errorf("searchsync: tags lesen: %w", err) + return nil, fmt.Errorf("searchsync: tags lesen: %w", err) } defer rows.Close() var names []string for rows.Next() { var name string if err := rows.Scan(&name); err != nil { - return "", fmt.Errorf("searchsync: tag-zeile lesen: %w", err) + return nil, fmt.Errorf("searchsync: tag-zeile lesen: %w", err) } names = append(names, name) } - return strings.Join(names, " "), rows.Err() + return names, rows.Err() +} + +func (s *Syncer) loadAttrs(ctx context.Context, documentID string, tagNames []string) (searchindex.Attrs, error) { + attrs := searchindex.Attrs{Tags: tagNames, Typed: map[string]float64{}, Strings: map[string]string{}} + rows, err := s.Pool.Query(ctx, ` + SELECT mf.field_key, mf.field_type, v.value + FROM document_metadata_values v JOIN metadata_fields mf ON mf.id = v.field_id + WHERE v.document_id = $1 + `, documentID) + if err != nil { + return attrs, fmt.Errorf("searchsync: metadatenwerte lesen: %w", err) + } + defer rows.Close() + for rows.Next() { + var key, typ, value string + if err := rows.Scan(&key, &typ, &value); err != nil { + return attrs, fmt.Errorf("searchsync: metadatenwert lesen: %w", err) + } + if !searchindex.ValidCustomKey(key) { + continue + } + attrs.Strings[key] = value + switch typ { + case "number": + if n, err := strconv.ParseFloat(value, 64); err == nil { + attrs.Typed[key] = n + } + case "date": + if d, err := time.Parse("2006-01-02", value); err == nil { + attrs.Typed[key] = float64(d.Unix()) + } + case "bool": + if b, err := strconv.ParseBool(value); err == nil && b { + attrs.Typed[key] = 1 + } else if err == nil { + attrs.Typed[key] = 0 + } + } + } + return attrs, rows.Err() } func (s *Syncer) loadFolderPath(ctx context.Context, folderID *string) (string, error) {