SRC-11: feld-whitelist-query-builder ohne dynamische sql-teile
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e1b58bfeb6
commit
1a70ffee81
9 files changed
+355
-121
No files matched your search
@@ -50,12 +50,6 @@ type responseJSON struct {
|
||||
Hits []hitJSON `json:"hits"`
|
||||
}
|
||||
|
||||
var sortByName = map[string]searchindex.Sort{
|
||||
"relevance": searchindex.SortRelevance,
|
||||
"updated": searchindex.SortUpdatedDesc,
|
||||
"created": searchindex.SortCreatedDesc,
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, message string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
@@ -105,18 +99,25 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadRequest, "ungueltiger parameter \"offset\"")
|
||||
return
|
||||
}
|
||||
sortKey := q.Get("sort")
|
||||
if sortKey == "" {
|
||||
sortKey = "relevance"
|
||||
}
|
||||
sortVal, ok := sortByName[sortKey]
|
||||
if !ok {
|
||||
writeError(w, http.StatusBadRequest, "ungueltiger parameter \"sort\" (relevance, updated, created)")
|
||||
sortVal, err := searchindex.ParseSort(q.Get("sort"))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
filters := []searchindex.Filter{{Field: searchindex.FilterCreatedBy, Value: caller}}
|
||||
for _, name := range []string{"document_type", "folder_path"} {
|
||||
if v := q.Get(name); v != "" {
|
||||
field, err := searchindex.ParseFilterField(name)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
filters = append(filters, searchindex.Filter{Field: field, Value: v})
|
||||
}
|
||||
}
|
||||
|
||||
page, err := searchindex.SearchPage(r.Context(), s.index, searchindex.SearchRequest{
|
||||
Term: term, OwnerID: caller, Sort: sortVal, Limit: limit, Offset: offset,
|
||||
Term: term, Filters: filters, Sort: sortVal, Limit: limit, Offset: offset,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, searchindex.ErrInvalidRequest) {
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.perlbach24.de/scripte/nexarch/dms/internal/searchindex"
|
||||
"gitea.perlbach24.de/scripte/nexarch/dms/internal/testlock"
|
||||
)
|
||||
|
||||
const responseTimeLimit = 500 * time.Millisecond
|
||||
@@ -31,6 +32,7 @@ func setup(t *testing.T, perms PermissionChecker) (*httptest.Server, *sql.DB) {
|
||||
if addr == "" {
|
||||
t.Skip("TEST_MANTICORE_ADDR nicht gesetzt, Manticore-Integrationstest uebersprungen")
|
||||
}
|
||||
testlock.Acquire(t, "manticore")
|
||||
db, err := searchindex.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
|
||||
@@ -5,14 +5,21 @@ import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Field ist ein geschlossener Aufzaehlungstyp der durchsuchbaren Spalten
|
||||
// (Akzeptanzkriterium 2: dokumentierte Gewichtung). Verhindert das aus den
|
||||
// Vorgaengerprojekten bekannte Problem dynamisch per Sprintf/Join gebauter
|
||||
// Spaltennamen in SQL — nur diese drei Werte koennen je in eine Abfrage
|
||||
// gelangen, ein beliebiger String kommt nicht bis zum SQL-Builder durch.
|
||||
// SQL-Fragmente entstehen ausschliesslich aus den Konstanten und
|
||||
// switch-Zweigen dieser Datei. Nutzereingaben erreichen die Abfrage nur als
|
||||
// Platzhalter-Argumente oder als typisierte Enum-Werte, die im switch auf
|
||||
// String-Literale abgebildet werden; ein unbekannter Wert endet im
|
||||
// default-Zweig mit ErrInvalidRequest. Es gibt keinen Pfad, auf dem ein
|
||||
// String von aussen als Spalten- oder Tabellenname in die Abfrage gelangt.
|
||||
// Sprintf/Join sind dafuer verboten (TestQueryBuilder_NoDynamicSQLAssembly).
|
||||
|
||||
var ErrInvalidRequest = errors.New("searchindex: ungueltige suchanfrage")
|
||||
|
||||
// Field sind die gewichteten Volltextfelder.
|
||||
type Field int
|
||||
|
||||
const (
|
||||
@@ -21,34 +28,108 @@ const (
|
||||
FieldTags
|
||||
)
|
||||
|
||||
// fieldWeights ordnet jedem Field sein BM25-Gewicht zu (Akzeptanzkriterium
|
||||
// 2). Titel zaehlt am staerksten, Tags mittel, Volltext (OCR/Inhalt) am
|
||||
// schwaechsten — dieselbe Reihenfolge wie bei paperless-ngx' Trefferliste.
|
||||
var fieldWeights = map[Field]struct {
|
||||
column string
|
||||
weight int
|
||||
}{
|
||||
FieldTitle: {"title", 10},
|
||||
FieldTags: {"tags", 5},
|
||||
FieldBody: {"body", 1},
|
||||
}
|
||||
|
||||
func (f Field) column() (string, bool) {
|
||||
w, ok := fieldWeights[f]
|
||||
return w.column, ok
|
||||
}
|
||||
|
||||
// fieldWeightOption baut die OPTION field_weights-Klausel aus dem
|
||||
// geschlossenen fieldWeights-Katalog — keine Nutzereingabe erreicht diese
|
||||
// Zeichenkette.
|
||||
func fieldWeightOption() string {
|
||||
parts := make([]string, 0, len(fieldWeights))
|
||||
for _, w := range fieldWeights {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", w.column, w.weight))
|
||||
func (f Field) weight() (int, error) {
|
||||
switch f {
|
||||
case FieldTitle:
|
||||
return 10, nil
|
||||
case FieldTags:
|
||||
return 5, nil
|
||||
case FieldBody:
|
||||
return 1, nil
|
||||
default:
|
||||
return 0, fmt.Errorf("%w: unbekanntes feld %d", ErrInvalidRequest, f)
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
// fieldWeightOption ist die feste OPTION field_weights-Klausel, abgeleitet
|
||||
// aus den Gewichten oben (Test stellt Gleichlauf sicher).
|
||||
func fieldWeightOption() string {
|
||||
return "title=10,tags=5,body=1"
|
||||
}
|
||||
|
||||
// FilterField sind die per Gleichheit filterbaren Attribute.
|
||||
type FilterField int
|
||||
|
||||
const (
|
||||
FilterDocumentType FilterField = iota
|
||||
FilterFolderPath
|
||||
FilterCreatedBy
|
||||
)
|
||||
|
||||
func (f FilterField) clause() (string, error) {
|
||||
switch f {
|
||||
case FilterDocumentType:
|
||||
return "document_type = ?", nil
|
||||
case FilterFolderPath:
|
||||
return "folder_path = ?", nil
|
||||
case FilterCreatedBy:
|
||||
return "created_by = ?", nil
|
||||
default:
|
||||
return "", fmt.Errorf("%w: unbekanntes filterfeld %d", ErrInvalidRequest, f)
|
||||
}
|
||||
}
|
||||
|
||||
// ParseFilterField bildet einen Nutzernamen (z.B. Query-Parameter) auf das
|
||||
// Enum ab; alles ausserhalb der Liste wird abgewiesen.
|
||||
func ParseFilterField(name string) (FilterField, error) {
|
||||
switch name {
|
||||
case "document_type":
|
||||
return FilterDocumentType, nil
|
||||
case "folder_path":
|
||||
return FilterFolderPath, nil
|
||||
default:
|
||||
return 0, fmt.Errorf("%w: filterfeld %q nicht erlaubt", ErrInvalidRequest, name)
|
||||
}
|
||||
}
|
||||
|
||||
type Filter struct {
|
||||
Field FilterField
|
||||
Value string
|
||||
}
|
||||
|
||||
type Sort int
|
||||
|
||||
const (
|
||||
SortRelevance Sort = iota
|
||||
SortUpdatedDesc
|
||||
SortCreatedDesc
|
||||
)
|
||||
|
||||
func (s Sort) clause() (string, error) {
|
||||
switch s {
|
||||
case SortRelevance:
|
||||
return "score DESC, updated_at DESC", nil
|
||||
case SortUpdatedDesc:
|
||||
return "updated_at DESC", nil
|
||||
case SortCreatedDesc:
|
||||
return "created_at DESC", nil
|
||||
default:
|
||||
return "", fmt.Errorf("%w: unbekannte sortierung %d", ErrInvalidRequest, s)
|
||||
}
|
||||
}
|
||||
|
||||
func ParseSort(name string) (Sort, error) {
|
||||
switch name {
|
||||
case "", "relevance":
|
||||
return SortRelevance, nil
|
||||
case "updated":
|
||||
return SortUpdatedDesc, nil
|
||||
case "created":
|
||||
return SortCreatedDesc, nil
|
||||
default:
|
||||
return 0, fmt.Errorf("%w: sortierung %q nicht erlaubt", ErrInvalidRequest, name)
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultPageSize = 20
|
||||
MaxPageSize = 100
|
||||
MaxWindow = 1000
|
||||
|
||||
snippetOpen = ""
|
||||
snippetClose = ""
|
||||
)
|
||||
|
||||
// escapeMatchTerm entschaerft Manticores erweiterte Abfragesyntax
|
||||
// (Sonderzeichen wie @!^$()<>~"/ haben dort Bedeutung), damit ein
|
||||
// Suchbegriff aus der UI nicht als Abfragesyntax interpretiert wird.
|
||||
@@ -64,17 +145,15 @@ func escapeMatchTerm(term string) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// BuildSearchQuery erzeugt eine parametrisierte MATCH-Abfrage ueber
|
||||
// TableName mit der Feldgewichtung aus fieldWeights. term wird escaped,
|
||||
// nicht in die SQL-Struktur interpoliert.
|
||||
// BuildSearchQuery erzeugt eine parametrisierte MATCH-Abfrage mit
|
||||
// Feldgewichtung. term wird escaped und als Argument uebergeben.
|
||||
func BuildSearchQuery(term string, limit int) (query string, args []any) {
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
limit = DefaultPageSize
|
||||
}
|
||||
query = fmt.Sprintf(
|
||||
`SELECT id, document_id, title, folder_path, document_type FROM %s WHERE MATCH(?) LIMIT %d OPTION field_weights=(%s)`,
|
||||
TableName, limit, fieldWeightOption(),
|
||||
)
|
||||
query = "SELECT id, document_id, title, folder_path, document_type FROM " + TableName +
|
||||
" WHERE MATCH(?) LIMIT " + strconv.Itoa(limit) +
|
||||
" OPTION field_weights=(" + fieldWeightOption() + ")"
|
||||
return query, []any{escapeMatchTerm(term)}
|
||||
}
|
||||
|
||||
@@ -125,10 +204,8 @@ type Doc struct {
|
||||
// Upsert legt ein Dokument im Index an oder ersetzt es (REPLACE INTO ist
|
||||
// bei RT-Tabellen der Upsert-Mechanismus ueber die id).
|
||||
func Upsert(ctx context.Context, db *sql.DB, d Doc) error {
|
||||
query := fmt.Sprintf(
|
||||
`REPLACE INTO %s (id, document_id, title, body, tags, folder_path, document_type, created_by, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?)`,
|
||||
TableName,
|
||||
)
|
||||
const query = "REPLACE INTO " + TableName +
|
||||
" (id, document_id, title, body, tags, folder_path, document_type, created_by, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?)"
|
||||
_, err := db.ExecContext(ctx, query,
|
||||
d.ID, d.DocumentID, d.Title, d.Body, d.Tags, d.FolderPath, d.DocumentType, d.CreatedBy, d.CreatedAtSec, d.UpdatedAtSec,
|
||||
)
|
||||
@@ -140,43 +217,36 @@ func Upsert(ctx context.Context, db *sql.DB, d Doc) error {
|
||||
|
||||
// Delete entfernt ein Dokument aus dem Index.
|
||||
func Delete(ctx context.Context, db *sql.DB, id int64) error {
|
||||
query := fmt.Sprintf(`DELETE FROM %s WHERE id = ?`, TableName)
|
||||
const query = "DELETE FROM " + TableName + " WHERE id = ?"
|
||||
if _, err := db.ExecContext(ctx, query, id); err != nil {
|
||||
return fmt.Errorf("searchindex: dokument entfernen: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var ErrInvalidRequest = errors.New("searchindex: ungueltige suchanfrage")
|
||||
|
||||
type Sort int
|
||||
|
||||
const (
|
||||
SortRelevance Sort = iota
|
||||
SortUpdatedDesc
|
||||
SortCreatedDesc
|
||||
)
|
||||
|
||||
var sortClauses = map[Sort]string{
|
||||
SortRelevance: "score DESC, updated_at DESC",
|
||||
SortUpdatedDesc: "updated_at DESC",
|
||||
SortCreatedDesc: "created_at DESC",
|
||||
// AllDocumentIDs liefert die document_id aller indexierten Dokumente
|
||||
// (Konsistenzpruefung, SRC-02).
|
||||
func AllDocumentIDs(ctx context.Context, db *sql.DB) ([]string, error) {
|
||||
const query = "SELECT document_id FROM " + TableName
|
||||
rows, err := db.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("searchindex: dokument-ids abfragen: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("searchindex: dokument-id lesen: %w", err)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultPageSize = 20
|
||||
MaxPageSize = 100
|
||||
MaxWindow = 1000
|
||||
|
||||
snippetOpen = ""
|
||||
snippetClose = ""
|
||||
)
|
||||
|
||||
// SearchRequest.OwnerID begrenzt Treffer auf selbst angelegte Dokumente
|
||||
// (created_by), solange es keine zentrale RBAC-Anbindung gibt.
|
||||
type SearchRequest struct {
|
||||
Term string
|
||||
OwnerID string
|
||||
Filters []Filter
|
||||
Sort Sort
|
||||
Limit int
|
||||
Offset int
|
||||
@@ -235,18 +305,26 @@ func BuildPageQuery(req SearchRequest) (query string, args []any, err error) {
|
||||
if req.Offset < 0 || req.Offset+req.Limit > MaxWindow {
|
||||
return "", nil, fmt.Errorf("%w: offset+limit muss zwischen 0 und %d liegen", ErrInvalidRequest, MaxWindow)
|
||||
}
|
||||
order, ok := sortClauses[req.Sort]
|
||||
if !ok {
|
||||
return "", nil, fmt.Errorf("%w: unbekannte sortierung %d", ErrInvalidRequest, req.Sort)
|
||||
order, err := req.Sort.clause()
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
query = fmt.Sprintf(
|
||||
`SELECT id, document_id, title, folder_path, document_type, updated_at, WEIGHT() AS score, `+
|
||||
`SNIPPET(body, QUERY(), 'limit=240', 'around=8', 'before_match=%s', 'after_match=%s') AS snippet `+
|
||||
`FROM %s WHERE MATCH(?) AND created_by = ? ORDER BY %s LIMIT %d, %d `+
|
||||
`OPTION field_weights=(%s), max_matches=%d`,
|
||||
snippetOpen, snippetClose, TableName, order, req.Offset, req.Limit, fieldWeightOption(), MaxWindow,
|
||||
)
|
||||
return query, []any{escapeMatchTerm(req.Term), req.OwnerID}, nil
|
||||
where := "MATCH(?)"
|
||||
args = []any{escapeMatchTerm(req.Term)}
|
||||
for _, f := range req.Filters {
|
||||
clause, err := f.Field.clause()
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
where += " AND " + clause
|
||||
args = append(args, f.Value)
|
||||
}
|
||||
query = "SELECT id, document_id, title, folder_path, document_type, updated_at, WEIGHT() AS score, " +
|
||||
"SNIPPET(body, QUERY(), 'limit=240', 'around=8', 'before_match=" + snippetOpen + "', 'after_match=" + snippetClose + "') AS snippet " +
|
||||
"FROM " + TableName + " WHERE " + where + " ORDER BY " + order +
|
||||
" LIMIT " + strconv.Itoa(req.Offset) + ", " + strconv.Itoa(req.Limit) +
|
||||
" OPTION field_weights=(" + fieldWeightOption() + "), max_matches=" + strconv.Itoa(MaxWindow)
|
||||
return query, args, nil
|
||||
}
|
||||
|
||||
func SearchPage(ctx context.Context, db *sql.DB, req SearchRequest) (Page, error) {
|
||||
|
||||
@@ -6,17 +6,19 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildPageQuery_UsesStaticSortClausesOnly(t *testing.T) {
|
||||
for sortKey, clause := range sortClauses {
|
||||
q, args, err := BuildPageQuery(SearchRequest{Term: "x", OwnerID: "u", Sort: sortKey})
|
||||
func TestBuildPageQuery_SortClausesAreStatic(t *testing.T) {
|
||||
want := map[Sort]string{
|
||||
SortRelevance: "ORDER BY score DESC, updated_at DESC ",
|
||||
SortUpdatedDesc: "ORDER BY updated_at DESC ",
|
||||
SortCreatedDesc: "ORDER BY created_at DESC ",
|
||||
}
|
||||
for s, clause := range want {
|
||||
q, _, err := BuildPageQuery(SearchRequest{Term: "x", Sort: s})
|
||||
if err != nil {
|
||||
t.Fatalf("sort %d: %v", sortKey, err)
|
||||
t.Fatalf("sort %d: %v", s, err)
|
||||
}
|
||||
if !strings.Contains(q, "ORDER BY "+clause+" ") {
|
||||
t.Errorf("sort %d: klausel %q fehlt in %q", sortKey, clause, q)
|
||||
}
|
||||
if len(args) != 2 {
|
||||
t.Errorf("args = %v, want term+owner", args)
|
||||
if !strings.Contains(q, clause) {
|
||||
t.Errorf("sort %d: %q fehlt in %q", s, clause, q)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -24,6 +26,7 @@ func TestBuildPageQuery_UsesStaticSortClausesOnly(t *testing.T) {
|
||||
func TestBuildPageQuery_RejectsInvalidRequests(t *testing.T) {
|
||||
cases := map[string]SearchRequest{
|
||||
"unbekannte sortierung": {Term: "x", Sort: Sort(99)},
|
||||
"unbekanntes filter": {Term: "x", Filters: []Filter{{Field: FilterField(99), Value: "v"}}},
|
||||
"limit zu gross": {Term: "x", Limit: MaxPageSize + 1},
|
||||
"negatives offset": {Term: "x", Offset: -1},
|
||||
"fenster zu gross": {Term: "x", Limit: 50, Offset: MaxWindow},
|
||||
@@ -35,16 +38,59 @@ func TestBuildPageQuery_RejectsInvalidRequests(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPageQuery_TermAndOwnerAreParametersNotSQL(t *testing.T) {
|
||||
q, args, err := BuildPageQuery(SearchRequest{Term: `a" OR 1=1 --`, OwnerID: "u' OR '1'='1"})
|
||||
func TestBuildPageQuery_ValuesAreParametersNotSQL(t *testing.T) {
|
||||
evil := `x' OR '1'='1`
|
||||
q, args, err := BuildPageQuery(SearchRequest{
|
||||
Term: `a" OR 1=1 --`,
|
||||
Filters: []Filter{{Field: FilterDocumentType, Value: evil}, {Field: FilterCreatedBy, Value: evil}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(q, "OR 1=1") || strings.Contains(q, "OR '1'") {
|
||||
if strings.Contains(q, "1=1") || strings.Contains(q, evil) {
|
||||
t.Errorf("nutzereingabe in sql-struktur: %q", q)
|
||||
}
|
||||
if args[1] != "u' OR '1'='1" {
|
||||
t.Errorf("owner arg = %v", args[1])
|
||||
if !strings.Contains(q, "document_type = ? AND created_by = ?") || len(args) != 3 || args[1] != evil {
|
||||
t.Errorf("filter nicht als platzhalter: q=%q args=%v", q, args)
|
||||
}
|
||||
}
|
||||
|
||||
// Pruefung 1: nicht erlaubte Felder werden abgewiesen, nicht ignoriert.
|
||||
func TestParse_RejectsFieldsOutsideWhitelist(t *testing.T) {
|
||||
for _, name := range []string{"body", "id", "document_type; DROP TABLE x", "DOCUMENT_TYPE", "created_by", ""} {
|
||||
if _, err := ParseFilterField(name); !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Errorf("ParseFilterField(%q) err = %v, want ErrInvalidRequest", name, err)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"title", "score", "updated_at DESC; --", "UPDATED"} {
|
||||
if _, err := ParseSort(name); !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Errorf("ParseSort(%q) err = %v, want ErrInvalidRequest", name, err)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"document_type", "folder_path"} {
|
||||
if _, err := ParseFilterField(name); err != nil {
|
||||
t.Errorf("ParseFilterField(%q): %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFieldWeightOption_MatchesFieldWeights(t *testing.T) {
|
||||
var parts []string
|
||||
for _, f := range []struct {
|
||||
field Field
|
||||
name string
|
||||
}{{FieldTitle, "title"}, {FieldTags, "tags"}, {FieldBody, "body"}} {
|
||||
w, err := f.field.weight()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parts = append(parts, f.name+"="+itoa(w))
|
||||
}
|
||||
if got := fieldWeightOption(); got != strings.Join(parts, ",") {
|
||||
t.Errorf("fieldWeightOption = %q, want %q", got, strings.Join(parts, ","))
|
||||
}
|
||||
if _, err := Field(99).weight(); !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Errorf("unbekanntes feld: err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.perlbach24.de/scripte/nexarch/dms/internal/testlock"
|
||||
)
|
||||
|
||||
// requireManticoreTestEnv liefert eine offene Verbindung zum echten
|
||||
@@ -17,6 +19,7 @@ func requireManticoreTestEnv(t *testing.T) *sql.DB {
|
||||
if addr == "" {
|
||||
t.Skip("TEST_MANTICORE_ADDR nicht gesetzt, Manticore-Integrationstest uebersprungen")
|
||||
}
|
||||
testlock.Acquire(t, "manticore")
|
||||
db, err := Open(addr)
|
||||
if err != nil {
|
||||
t.Fatalf("verbindung aufbauen: %v", err)
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package searchindex
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func itoa(i int) string { return strconv.Itoa(i) }
|
||||
|
||||
// Pruefung 2 (automatisiert): kein dynamischer Spalten-/Tabellenname per
|
||||
// String-Zusammenbau. Sprintf-Familie ist im Paket ausserhalb von Open (DSN)
|
||||
// verboten, und in jeder String-Verkettung duerfen nur Literale, die hier
|
||||
// gelisteten Konstanten und strconv.Itoa-Aufrufe stehen.
|
||||
func TestQueryBuilder_NoDynamicSQLAssembly(t *testing.T) {
|
||||
allowedConsts := map[string]bool{
|
||||
"TableName": true, "snippetOpen": true, "snippetClose": true, "MaxWindow": true,
|
||||
"where": true, "order": true, "clause": true,
|
||||
}
|
||||
fset := token.NewFileSet()
|
||||
files := []string{"query.go", "schema.go"}
|
||||
for _, name := range files {
|
||||
f, err := parser.ParseFile(fset, name, nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
for _, decl := range f.Decls {
|
||||
fn, isFn := decl.(*ast.FuncDecl)
|
||||
ast.Inspect(decl, func(n ast.Node) bool {
|
||||
switch x := n.(type) {
|
||||
case *ast.CallExpr:
|
||||
sel, ok := x.Fun.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
pkg, ok := sel.X.(*ast.Ident)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if pkg.Name == "fmt" && (sel.Sel.Name == "Sprintf" || sel.Sel.Name == "Sprint" || sel.Sel.Name == "Sprintln") {
|
||||
if !(isFn && fn.Name.Name == "Open") {
|
||||
t.Errorf("%s: fmt.%s ausserhalb von Open", fset.Position(x.Pos()), sel.Sel.Name)
|
||||
}
|
||||
}
|
||||
if pkg.Name == "strings" && sel.Sel.Name == "Join" && isFn && fn.Name.Name != "escapeMatchTerm" {
|
||||
t.Errorf("%s: strings.Join im Query-Builder", fset.Position(x.Pos()))
|
||||
}
|
||||
case *ast.BinaryExpr:
|
||||
if x.Op != token.ADD {
|
||||
return true
|
||||
}
|
||||
for _, operand := range []ast.Expr{x.X, x.Y} {
|
||||
switch o := operand.(type) {
|
||||
case *ast.BasicLit, *ast.BinaryExpr:
|
||||
case *ast.Ident:
|
||||
if !allowedConsts[o.Name] {
|
||||
t.Errorf("%s: unerlaubter operand %q in verkettung", fset.Position(o.Pos()), o.Name)
|
||||
}
|
||||
case *ast.CallExpr:
|
||||
sel, ok := o.Fun.(*ast.SelectorExpr)
|
||||
if ok {
|
||||
if p, ok := sel.X.(*ast.Ident); ok && p.Name == "strconv" && sel.Sel.Name == "Itoa" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
t.Errorf("%s: unerlaubter aufruf %s in verkettung", fset.Position(o.Pos()), sel.Sel.Name)
|
||||
} else {
|
||||
t.Errorf("%s: unerlaubter aufruf in verkettung", fset.Position(o.Pos()))
|
||||
}
|
||||
default:
|
||||
t.Errorf("%s: unerlaubter operand-typ %T in verkettung", fset.Position(operand.Pos()), operand)
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -296,22 +296,14 @@ func (s *Syncer) CheckConsistency(ctx context.Context) (ConsistencyReport, error
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
indexRows, err := s.Index.QueryContext(ctx, fmt.Sprintf(`SELECT document_id FROM %s`, searchindex.TableName))
|
||||
indexDocIDs, err := searchindex.AllDocumentIDs(ctx, s.Index)
|
||||
if err != nil {
|
||||
return ConsistencyReport{}, fmt.Errorf("searchsync: index-dokumente abfragen: %w", err)
|
||||
}
|
||||
defer indexRows.Close()
|
||||
indexIDs := map[string]bool{}
|
||||
for indexRows.Next() {
|
||||
var id string
|
||||
if err := indexRows.Scan(&id); err != nil {
|
||||
return ConsistencyReport{}, fmt.Errorf("searchsync: index-dokument-id lesen: %w", err)
|
||||
}
|
||||
indexIDs[id] = true
|
||||
}
|
||||
if err := indexRows.Err(); err != nil {
|
||||
return ConsistencyReport{}, err
|
||||
}
|
||||
indexIDs := make(map[string]bool, len(indexDocIDs))
|
||||
for _, id := range indexDocIDs {
|
||||
indexIDs[id] = true
|
||||
}
|
||||
|
||||
var report ConsistencyReport
|
||||
for id := range dbIDs {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"gitea.perlbach24.de/scripte/nexarch/dms/internal/jobqueue"
|
||||
"gitea.perlbach24.de/scripte/nexarch/dms/internal/searchindex"
|
||||
"gitea.perlbach24.de/scripte/nexarch/dms/internal/testlock"
|
||||
)
|
||||
|
||||
// testEnv setzt eine Minimalversion des Dokumentenschemas auf (dasselbe
|
||||
@@ -28,6 +29,7 @@ func testEnv(t *testing.T) (*pgxpool.Pool, *sql.DB, *jobqueue.Queue) {
|
||||
if manticoreAddr == "" {
|
||||
t.Skip("TEST_MANTICORE_ADDR nicht gesetzt, Integrationstest uebersprungen")
|
||||
}
|
||||
testlock.Acquire(t, "manticore")
|
||||
ctx := context.Background()
|
||||
|
||||
pool, err := pgxpool.New(ctx, dsn)
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package testlock
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Acquire serialisiert Tests, die sich eine externe Ressource (z.B. den
|
||||
// Manticore-Index) teilen, ueber Pakete, Prozesse und parallele Checkouts
|
||||
// desselben Hosts hinweg. Freigabe erfolgt automatisch in t.Cleanup.
|
||||
func Acquire(t *testing.T, resource string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(os.TempDir(), "nexarch-dms-test-"+resource+".lock")
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o666)
|
||||
if err != nil {
|
||||
t.Fatalf("testlock %s: %v", resource, err)
|
||||
}
|
||||
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
|
||||
_ = f.Close()
|
||||
t.Fatalf("testlock %s: %v", resource, err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
|
||||
_ = f.Close()
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user