From 1a70ffee81b22c8a080e2ed1998b3b38bcd10aeb Mon Sep 17 00:00:00 2001 From: patrick Date: Fri, 18 Sep 2026 20:45:32 +0200 Subject: [PATCH] SRC-11: feld-whitelist-query-builder ohne dynamische sql-teile Co-Authored-By: Claude Sonnet 5 --- dms/internal/searchapi/handler.go | 29 +-- dms/internal/searchapi/handler_test.go | 2 + dms/internal/searchindex/query.go | 238 ++++++++++++------- dms/internal/searchindex/query_page_test.go | 74 ++++-- dms/internal/searchindex/searchindex_test.go | 3 + dms/internal/searchindex/static_sql_test.go | 82 +++++++ dms/internal/searchsync/searchsync.go | 18 +- dms/internal/searchsync/searchsync_test.go | 2 + dms/internal/testlock/testlock.go | 28 +++ 9 files changed, 355 insertions(+), 121 deletions(-) create mode 100644 dms/internal/searchindex/static_sql_test.go create mode 100644 dms/internal/testlock/testlock.go diff --git a/dms/internal/searchapi/handler.go b/dms/internal/searchapi/handler.go index 9346c39e..7abc6327 100644 --- a/dms/internal/searchapi/handler.go +++ b/dms/internal/searchapi/handler.go @@ -50,12 +50,6 @@ type responseJSON struct { Hits []hitJSON `json:"hits"` } -var sortByName = map[string]searchindex.Sort{ - "relevance": searchindex.SortRelevance, - "updated": searchindex.SortUpdatedDesc, - "created": searchindex.SortCreatedDesc, -} - func writeError(w http.ResponseWriter, status int, message string) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) @@ -105,18 +99,25 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "ungueltiger parameter \"offset\"") return } - sortKey := q.Get("sort") - if sortKey == "" { - sortKey = "relevance" - } - sortVal, ok := sortByName[sortKey] - if !ok { - writeError(w, http.StatusBadRequest, "ungueltiger parameter \"sort\" (relevance, updated, created)") + sortVal, err := searchindex.ParseSort(q.Get("sort")) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) return } + filters := []searchindex.Filter{{Field: searchindex.FilterCreatedBy, Value: caller}} + for _, name := range []string{"document_type", "folder_path"} { + if v := q.Get(name); v != "" { + field, err := searchindex.ParseFilterField(name) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + filters = append(filters, searchindex.Filter{Field: field, Value: v}) + } + } page, err := searchindex.SearchPage(r.Context(), s.index, searchindex.SearchRequest{ - Term: term, OwnerID: caller, Sort: sortVal, Limit: limit, Offset: offset, + Term: term, Filters: filters, Sort: sortVal, Limit: limit, Offset: offset, }) if err != nil { if errors.Is(err, searchindex.ErrInvalidRequest) { diff --git a/dms/internal/searchapi/handler_test.go b/dms/internal/searchapi/handler_test.go index c5e51b50..c1680f63 100644 --- a/dms/internal/searchapi/handler_test.go +++ b/dms/internal/searchapi/handler_test.go @@ -15,6 +15,7 @@ import ( "time" "gitea.perlbach24.de/scripte/nexarch/dms/internal/searchindex" + "gitea.perlbach24.de/scripte/nexarch/dms/internal/testlock" ) const responseTimeLimit = 500 * time.Millisecond @@ -31,6 +32,7 @@ func setup(t *testing.T, perms PermissionChecker) (*httptest.Server, *sql.DB) { if addr == "" { t.Skip("TEST_MANTICORE_ADDR nicht gesetzt, Manticore-Integrationstest uebersprungen") } + testlock.Acquire(t, "manticore") db, err := searchindex.Open(addr) if err != nil { t.Fatalf("open: %v", err) diff --git a/dms/internal/searchindex/query.go b/dms/internal/searchindex/query.go index 19947e8d..d75e0d58 100644 --- a/dms/internal/searchindex/query.go +++ b/dms/internal/searchindex/query.go @@ -5,14 +5,21 @@ import ( "database/sql" "errors" "fmt" + "strconv" "strings" ) -// Field ist ein geschlossener Aufzaehlungstyp der durchsuchbaren Spalten -// (Akzeptanzkriterium 2: dokumentierte Gewichtung). Verhindert das aus den -// Vorgaengerprojekten bekannte Problem dynamisch per Sprintf/Join gebauter -// Spaltennamen in SQL — nur diese drei Werte koennen je in eine Abfrage -// gelangen, ein beliebiger String kommt nicht bis zum SQL-Builder durch. +// SQL-Fragmente entstehen ausschliesslich aus den Konstanten und +// switch-Zweigen dieser Datei. Nutzereingaben erreichen die Abfrage nur als +// Platzhalter-Argumente oder als typisierte Enum-Werte, die im switch auf +// String-Literale abgebildet werden; ein unbekannter Wert endet im +// default-Zweig mit ErrInvalidRequest. Es gibt keinen Pfad, auf dem ein +// String von aussen als Spalten- oder Tabellenname in die Abfrage gelangt. +// Sprintf/Join sind dafuer verboten (TestQueryBuilder_NoDynamicSQLAssembly). + +var ErrInvalidRequest = errors.New("searchindex: ungueltige suchanfrage") + +// Field sind die gewichteten Volltextfelder. type Field int const ( @@ -21,34 +28,108 @@ const ( FieldTags ) -// fieldWeights ordnet jedem Field sein BM25-Gewicht zu (Akzeptanzkriterium -// 2). Titel zaehlt am staerksten, Tags mittel, Volltext (OCR/Inhalt) am -// schwaechsten — dieselbe Reihenfolge wie bei paperless-ngx' Trefferliste. -var fieldWeights = map[Field]struct { - column string - weight int -}{ - FieldTitle: {"title", 10}, - FieldTags: {"tags", 5}, - FieldBody: {"body", 1}, -} - -func (f Field) column() (string, bool) { - w, ok := fieldWeights[f] - return w.column, ok -} - -// fieldWeightOption baut die OPTION field_weights-Klausel aus dem -// geschlossenen fieldWeights-Katalog — keine Nutzereingabe erreicht diese -// Zeichenkette. -func fieldWeightOption() string { - parts := make([]string, 0, len(fieldWeights)) - for _, w := range fieldWeights { - parts = append(parts, fmt.Sprintf("%s=%d", w.column, w.weight)) +func (f Field) weight() (int, error) { + switch f { + case FieldTitle: + return 10, nil + case FieldTags: + return 5, nil + case FieldBody: + return 1, nil + default: + return 0, fmt.Errorf("%w: unbekanntes feld %d", ErrInvalidRequest, f) } - return strings.Join(parts, ",") } +// fieldWeightOption ist die feste OPTION field_weights-Klausel, abgeleitet +// aus den Gewichten oben (Test stellt Gleichlauf sicher). +func fieldWeightOption() string { + return "title=10,tags=5,body=1" +} + +// FilterField sind die per Gleichheit filterbaren Attribute. +type FilterField int + +const ( + FilterDocumentType FilterField = iota + FilterFolderPath + FilterCreatedBy +) + +func (f FilterField) clause() (string, error) { + switch f { + case FilterDocumentType: + return "document_type = ?", nil + case FilterFolderPath: + return "folder_path = ?", nil + case FilterCreatedBy: + return "created_by = ?", nil + default: + return "", fmt.Errorf("%w: unbekanntes filterfeld %d", ErrInvalidRequest, f) + } +} + +// ParseFilterField bildet einen Nutzernamen (z.B. Query-Parameter) auf das +// Enum ab; alles ausserhalb der Liste wird abgewiesen. +func ParseFilterField(name string) (FilterField, error) { + switch name { + case "document_type": + return FilterDocumentType, nil + case "folder_path": + return FilterFolderPath, nil + default: + return 0, fmt.Errorf("%w: filterfeld %q nicht erlaubt", ErrInvalidRequest, name) + } +} + +type Filter struct { + Field FilterField + Value string +} + +type Sort int + +const ( + SortRelevance Sort = iota + SortUpdatedDesc + SortCreatedDesc +) + +func (s Sort) clause() (string, error) { + switch s { + case SortRelevance: + return "score DESC, updated_at DESC", nil + case SortUpdatedDesc: + return "updated_at DESC", nil + case SortCreatedDesc: + return "created_at DESC", nil + default: + return "", fmt.Errorf("%w: unbekannte sortierung %d", ErrInvalidRequest, s) + } +} + +func ParseSort(name string) (Sort, error) { + switch name { + case "", "relevance": + return SortRelevance, nil + case "updated": + return SortUpdatedDesc, nil + case "created": + return SortCreatedDesc, nil + default: + return 0, fmt.Errorf("%w: sortierung %q nicht erlaubt", ErrInvalidRequest, name) + } +} + +const ( + DefaultPageSize = 20 + MaxPageSize = 100 + MaxWindow = 1000 + + snippetOpen = "" + snippetClose = "" +) + // escapeMatchTerm entschaerft Manticores erweiterte Abfragesyntax // (Sonderzeichen wie @!^$()<>~"/ haben dort Bedeutung), damit ein // Suchbegriff aus der UI nicht als Abfragesyntax interpretiert wird. @@ -64,17 +145,15 @@ func escapeMatchTerm(term string) string { return b.String() } -// BuildSearchQuery erzeugt eine parametrisierte MATCH-Abfrage ueber -// TableName mit der Feldgewichtung aus fieldWeights. term wird escaped, -// nicht in die SQL-Struktur interpoliert. +// BuildSearchQuery erzeugt eine parametrisierte MATCH-Abfrage mit +// Feldgewichtung. term wird escaped und als Argument uebergeben. func BuildSearchQuery(term string, limit int) (query string, args []any) { if limit <= 0 { - limit = 20 + limit = DefaultPageSize } - query = fmt.Sprintf( - `SELECT id, document_id, title, folder_path, document_type FROM %s WHERE MATCH(?) LIMIT %d OPTION field_weights=(%s)`, - TableName, limit, fieldWeightOption(), - ) + query = "SELECT id, document_id, title, folder_path, document_type FROM " + TableName + + " WHERE MATCH(?) LIMIT " + strconv.Itoa(limit) + + " OPTION field_weights=(" + fieldWeightOption() + ")" return query, []any{escapeMatchTerm(term)} } @@ -125,10 +204,8 @@ type Doc struct { // Upsert legt ein Dokument im Index an oder ersetzt es (REPLACE INTO ist // bei RT-Tabellen der Upsert-Mechanismus ueber die id). func Upsert(ctx context.Context, db *sql.DB, d Doc) error { - query := fmt.Sprintf( - `REPLACE INTO %s (id, document_id, title, body, tags, folder_path, document_type, created_by, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?)`, - TableName, - ) + const query = "REPLACE INTO " + TableName + + " (id, document_id, title, body, tags, folder_path, document_type, created_by, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?)" _, err := db.ExecContext(ctx, query, d.ID, d.DocumentID, d.Title, d.Body, d.Tags, d.FolderPath, d.DocumentType, d.CreatedBy, d.CreatedAtSec, d.UpdatedAtSec, ) @@ -140,43 +217,36 @@ func Upsert(ctx context.Context, db *sql.DB, d Doc) error { // Delete entfernt ein Dokument aus dem Index. func Delete(ctx context.Context, db *sql.DB, id int64) error { - query := fmt.Sprintf(`DELETE FROM %s WHERE id = ?`, TableName) + const query = "DELETE FROM " + TableName + " WHERE id = ?" if _, err := db.ExecContext(ctx, query, id); err != nil { return fmt.Errorf("searchindex: dokument entfernen: %w", err) } return nil } -var ErrInvalidRequest = errors.New("searchindex: ungueltige suchanfrage") - -type Sort int - -const ( - SortRelevance Sort = iota - SortUpdatedDesc - SortCreatedDesc -) - -var sortClauses = map[Sort]string{ - SortRelevance: "score DESC, updated_at DESC", - SortUpdatedDesc: "updated_at DESC", - SortCreatedDesc: "created_at DESC", +// AllDocumentIDs liefert die document_id aller indexierten Dokumente +// (Konsistenzpruefung, SRC-02). +func AllDocumentIDs(ctx context.Context, db *sql.DB) ([]string, error) { + const query = "SELECT document_id FROM " + TableName + rows, err := db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("searchindex: dokument-ids abfragen: %w", err) + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("searchindex: dokument-id lesen: %w", err) + } + ids = append(ids, id) + } + return ids, rows.Err() } -const ( - DefaultPageSize = 20 - MaxPageSize = 100 - MaxWindow = 1000 - - snippetOpen = "" - snippetClose = "" -) - -// SearchRequest.OwnerID begrenzt Treffer auf selbst angelegte Dokumente -// (created_by), solange es keine zentrale RBAC-Anbindung gibt. type SearchRequest struct { Term string - OwnerID string + Filters []Filter Sort Sort Limit int Offset int @@ -235,18 +305,26 @@ func BuildPageQuery(req SearchRequest) (query string, args []any, err error) { if req.Offset < 0 || req.Offset+req.Limit > MaxWindow { return "", nil, fmt.Errorf("%w: offset+limit muss zwischen 0 und %d liegen", ErrInvalidRequest, MaxWindow) } - order, ok := sortClauses[req.Sort] - if !ok { - return "", nil, fmt.Errorf("%w: unbekannte sortierung %d", ErrInvalidRequest, req.Sort) + order, err := req.Sort.clause() + if err != nil { + return "", nil, err } - query = fmt.Sprintf( - `SELECT id, document_id, title, folder_path, document_type, updated_at, WEIGHT() AS score, `+ - `SNIPPET(body, QUERY(), 'limit=240', 'around=8', 'before_match=%s', 'after_match=%s') AS snippet `+ - `FROM %s WHERE MATCH(?) AND created_by = ? ORDER BY %s LIMIT %d, %d `+ - `OPTION field_weights=(%s), max_matches=%d`, - snippetOpen, snippetClose, TableName, order, req.Offset, req.Limit, fieldWeightOption(), MaxWindow, - ) - return query, []any{escapeMatchTerm(req.Term), req.OwnerID}, nil + where := "MATCH(?)" + args = []any{escapeMatchTerm(req.Term)} + for _, f := range req.Filters { + clause, err := f.Field.clause() + if err != nil { + return "", nil, err + } + where += " AND " + clause + args = append(args, f.Value) + } + query = "SELECT id, document_id, title, folder_path, document_type, updated_at, WEIGHT() AS score, " + + "SNIPPET(body, QUERY(), 'limit=240', 'around=8', 'before_match=" + snippetOpen + "', 'after_match=" + snippetClose + "') AS snippet " + + "FROM " + TableName + " WHERE " + where + " ORDER BY " + order + + " LIMIT " + strconv.Itoa(req.Offset) + ", " + strconv.Itoa(req.Limit) + + " OPTION field_weights=(" + fieldWeightOption() + "), max_matches=" + strconv.Itoa(MaxWindow) + return query, args, nil } func SearchPage(ctx context.Context, db *sql.DB, req SearchRequest) (Page, error) { diff --git a/dms/internal/searchindex/query_page_test.go b/dms/internal/searchindex/query_page_test.go index dcbd3922..a1a1de11 100644 --- a/dms/internal/searchindex/query_page_test.go +++ b/dms/internal/searchindex/query_page_test.go @@ -6,17 +6,19 @@ import ( "testing" ) -func TestBuildPageQuery_UsesStaticSortClausesOnly(t *testing.T) { - for sortKey, clause := range sortClauses { - q, args, err := BuildPageQuery(SearchRequest{Term: "x", OwnerID: "u", Sort: sortKey}) +func TestBuildPageQuery_SortClausesAreStatic(t *testing.T) { + want := map[Sort]string{ + SortRelevance: "ORDER BY score DESC, updated_at DESC ", + SortUpdatedDesc: "ORDER BY updated_at DESC ", + SortCreatedDesc: "ORDER BY created_at DESC ", + } + for s, clause := range want { + q, _, err := BuildPageQuery(SearchRequest{Term: "x", Sort: s}) if err != nil { - t.Fatalf("sort %d: %v", sortKey, err) + t.Fatalf("sort %d: %v", s, err) } - if !strings.Contains(q, "ORDER BY "+clause+" ") { - t.Errorf("sort %d: klausel %q fehlt in %q", sortKey, clause, q) - } - if len(args) != 2 { - t.Errorf("args = %v, want term+owner", args) + if !strings.Contains(q, clause) { + t.Errorf("sort %d: %q fehlt in %q", s, clause, q) } } } @@ -24,6 +26,7 @@ func TestBuildPageQuery_UsesStaticSortClausesOnly(t *testing.T) { func TestBuildPageQuery_RejectsInvalidRequests(t *testing.T) { cases := map[string]SearchRequest{ "unbekannte sortierung": {Term: "x", Sort: Sort(99)}, + "unbekanntes filter": {Term: "x", Filters: []Filter{{Field: FilterField(99), Value: "v"}}}, "limit zu gross": {Term: "x", Limit: MaxPageSize + 1}, "negatives offset": {Term: "x", Offset: -1}, "fenster zu gross": {Term: "x", Limit: 50, Offset: MaxWindow}, @@ -35,16 +38,59 @@ func TestBuildPageQuery_RejectsInvalidRequests(t *testing.T) { } } -func TestBuildPageQuery_TermAndOwnerAreParametersNotSQL(t *testing.T) { - q, args, err := BuildPageQuery(SearchRequest{Term: `a" OR 1=1 --`, OwnerID: "u' OR '1'='1"}) +func TestBuildPageQuery_ValuesAreParametersNotSQL(t *testing.T) { + evil := `x' OR '1'='1` + q, args, err := BuildPageQuery(SearchRequest{ + Term: `a" OR 1=1 --`, + Filters: []Filter{{Field: FilterDocumentType, Value: evil}, {Field: FilterCreatedBy, Value: evil}}, + }) if err != nil { t.Fatal(err) } - if strings.Contains(q, "OR 1=1") || strings.Contains(q, "OR '1'") { + if strings.Contains(q, "1=1") || strings.Contains(q, evil) { t.Errorf("nutzereingabe in sql-struktur: %q", q) } - if args[1] != "u' OR '1'='1" { - t.Errorf("owner arg = %v", args[1]) + if !strings.Contains(q, "document_type = ? AND created_by = ?") || len(args) != 3 || args[1] != evil { + t.Errorf("filter nicht als platzhalter: q=%q args=%v", q, args) + } +} + +// Pruefung 1: nicht erlaubte Felder werden abgewiesen, nicht ignoriert. +func TestParse_RejectsFieldsOutsideWhitelist(t *testing.T) { + for _, name := range []string{"body", "id", "document_type; DROP TABLE x", "DOCUMENT_TYPE", "created_by", ""} { + if _, err := ParseFilterField(name); !errors.Is(err, ErrInvalidRequest) { + t.Errorf("ParseFilterField(%q) err = %v, want ErrInvalidRequest", name, err) + } + } + for _, name := range []string{"title", "score", "updated_at DESC; --", "UPDATED"} { + if _, err := ParseSort(name); !errors.Is(err, ErrInvalidRequest) { + t.Errorf("ParseSort(%q) err = %v, want ErrInvalidRequest", name, err) + } + } + for _, name := range []string{"document_type", "folder_path"} { + if _, err := ParseFilterField(name); err != nil { + t.Errorf("ParseFilterField(%q): %v", name, err) + } + } +} + +func TestFieldWeightOption_MatchesFieldWeights(t *testing.T) { + var parts []string + for _, f := range []struct { + field Field + name string + }{{FieldTitle, "title"}, {FieldTags, "tags"}, {FieldBody, "body"}} { + w, err := f.field.weight() + if err != nil { + t.Fatal(err) + } + parts = append(parts, f.name+"="+itoa(w)) + } + if got := fieldWeightOption(); got != strings.Join(parts, ",") { + t.Errorf("fieldWeightOption = %q, want %q", got, strings.Join(parts, ",")) + } + if _, err := Field(99).weight(); !errors.Is(err, ErrInvalidRequest) { + t.Errorf("unbekanntes feld: err = %v", err) } } diff --git a/dms/internal/searchindex/searchindex_test.go b/dms/internal/searchindex/searchindex_test.go index a15e4b1b..55f86165 100644 --- a/dms/internal/searchindex/searchindex_test.go +++ b/dms/internal/searchindex/searchindex_test.go @@ -6,6 +6,8 @@ import ( "os" "strings" "testing" + + "gitea.perlbach24.de/scripte/nexarch/dms/internal/testlock" ) // requireManticoreTestEnv liefert eine offene Verbindung zum echten @@ -17,6 +19,7 @@ func requireManticoreTestEnv(t *testing.T) *sql.DB { if addr == "" { t.Skip("TEST_MANTICORE_ADDR nicht gesetzt, Manticore-Integrationstest uebersprungen") } + testlock.Acquire(t, "manticore") db, err := Open(addr) if err != nil { t.Fatalf("verbindung aufbauen: %v", err) diff --git a/dms/internal/searchindex/static_sql_test.go b/dms/internal/searchindex/static_sql_test.go new file mode 100644 index 00000000..a611d7a8 --- /dev/null +++ b/dms/internal/searchindex/static_sql_test.go @@ -0,0 +1,82 @@ +package searchindex + +import ( + "go/ast" + "go/parser" + "go/token" + "strconv" + "testing" +) + +func itoa(i int) string { return strconv.Itoa(i) } + +// Pruefung 2 (automatisiert): kein dynamischer Spalten-/Tabellenname per +// String-Zusammenbau. Sprintf-Familie ist im Paket ausserhalb von Open (DSN) +// verboten, und in jeder String-Verkettung duerfen nur Literale, die hier +// gelisteten Konstanten und strconv.Itoa-Aufrufe stehen. +func TestQueryBuilder_NoDynamicSQLAssembly(t *testing.T) { + allowedConsts := map[string]bool{ + "TableName": true, "snippetOpen": true, "snippetClose": true, "MaxWindow": true, + "where": true, "order": true, "clause": true, + } + fset := token.NewFileSet() + files := []string{"query.go", "schema.go"} + for _, name := range files { + f, err := parser.ParseFile(fset, name, nil, 0) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + for _, decl := range f.Decls { + fn, isFn := decl.(*ast.FuncDecl) + ast.Inspect(decl, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.CallExpr: + sel, ok := x.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + pkg, ok := sel.X.(*ast.Ident) + if !ok { + return true + } + if pkg.Name == "fmt" && (sel.Sel.Name == "Sprintf" || sel.Sel.Name == "Sprint" || sel.Sel.Name == "Sprintln") { + if !(isFn && fn.Name.Name == "Open") { + t.Errorf("%s: fmt.%s ausserhalb von Open", fset.Position(x.Pos()), sel.Sel.Name) + } + } + if pkg.Name == "strings" && sel.Sel.Name == "Join" && isFn && fn.Name.Name != "escapeMatchTerm" { + t.Errorf("%s: strings.Join im Query-Builder", fset.Position(x.Pos())) + } + case *ast.BinaryExpr: + if x.Op != token.ADD { + return true + } + for _, operand := range []ast.Expr{x.X, x.Y} { + switch o := operand.(type) { + case *ast.BasicLit, *ast.BinaryExpr: + case *ast.Ident: + if !allowedConsts[o.Name] { + t.Errorf("%s: unerlaubter operand %q in verkettung", fset.Position(o.Pos()), o.Name) + } + case *ast.CallExpr: + sel, ok := o.Fun.(*ast.SelectorExpr) + if ok { + if p, ok := sel.X.(*ast.Ident); ok && p.Name == "strconv" && sel.Sel.Name == "Itoa" { + continue + } + } + if ok { + t.Errorf("%s: unerlaubter aufruf %s in verkettung", fset.Position(o.Pos()), sel.Sel.Name) + } else { + t.Errorf("%s: unerlaubter aufruf in verkettung", fset.Position(o.Pos())) + } + default: + t.Errorf("%s: unerlaubter operand-typ %T in verkettung", fset.Position(operand.Pos()), operand) + } + } + } + return true + }) + } + } +} diff --git a/dms/internal/searchsync/searchsync.go b/dms/internal/searchsync/searchsync.go index 6b2f342a..530d2391 100644 --- a/dms/internal/searchsync/searchsync.go +++ b/dms/internal/searchsync/searchsync.go @@ -296,22 +296,14 @@ func (s *Syncer) CheckConsistency(ctx context.Context) (ConsistencyReport, error } rows.Close() - indexRows, err := s.Index.QueryContext(ctx, fmt.Sprintf(`SELECT document_id FROM %s`, searchindex.TableName)) + indexDocIDs, err := searchindex.AllDocumentIDs(ctx, s.Index) if err != nil { - return ConsistencyReport{}, fmt.Errorf("searchsync: index-dokumente abfragen: %w", err) - } - defer indexRows.Close() - indexIDs := map[string]bool{} - for indexRows.Next() { - var id string - if err := indexRows.Scan(&id); err != nil { - return ConsistencyReport{}, fmt.Errorf("searchsync: index-dokument-id lesen: %w", err) - } - indexIDs[id] = true - } - if err := indexRows.Err(); err != nil { return ConsistencyReport{}, err } + indexIDs := make(map[string]bool, len(indexDocIDs)) + for _, id := range indexDocIDs { + indexIDs[id] = true + } var report ConsistencyReport for id := range dbIDs { diff --git a/dms/internal/searchsync/searchsync_test.go b/dms/internal/searchsync/searchsync_test.go index 7adb6575..26865986 100644 --- a/dms/internal/searchsync/searchsync_test.go +++ b/dms/internal/searchsync/searchsync_test.go @@ -11,6 +11,7 @@ import ( "gitea.perlbach24.de/scripte/nexarch/dms/internal/jobqueue" "gitea.perlbach24.de/scripte/nexarch/dms/internal/searchindex" + "gitea.perlbach24.de/scripte/nexarch/dms/internal/testlock" ) // testEnv setzt eine Minimalversion des Dokumentenschemas auf (dasselbe @@ -28,6 +29,7 @@ func testEnv(t *testing.T) (*pgxpool.Pool, *sql.DB, *jobqueue.Queue) { if manticoreAddr == "" { t.Skip("TEST_MANTICORE_ADDR nicht gesetzt, Integrationstest uebersprungen") } + testlock.Acquire(t, "manticore") ctx := context.Background() pool, err := pgxpool.New(ctx, dsn) diff --git a/dms/internal/testlock/testlock.go b/dms/internal/testlock/testlock.go new file mode 100644 index 00000000..66e2042f --- /dev/null +++ b/dms/internal/testlock/testlock.go @@ -0,0 +1,28 @@ +package testlock + +import ( + "os" + "path/filepath" + "syscall" + "testing" +) + +// Acquire serialisiert Tests, die sich eine externe Ressource (z.B. den +// Manticore-Index) teilen, ueber Pakete, Prozesse und parallele Checkouts +// desselben Hosts hinweg. Freigabe erfolgt automatisch in t.Cleanup. +func Acquire(t *testing.T, resource string) { + t.Helper() + path := filepath.Join(os.TempDir(), "nexarch-dms-test-"+resource+".lock") + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o666) + if err != nil { + t.Fatalf("testlock %s: %v", resource, err) + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { + _ = f.Close() + t.Fatalf("testlock %s: %v", resource, err) + } + t.Cleanup(func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + _ = f.Close() + }) +}