FDN-01: repository & projektgerüst
Git-Repository für bestehenden archivdms-Code initialisiert, Branch-/Commit-Konvention (feature/<ticket>-<slug>-Branches, Ticket-Prefix in Commit-Nachricht) etabliert.
This commit is contained in:
@@ -0,0 +1,94 @@
|
||||
// Package index is the (Phase 1) full-text search sync layer for archivdms.
|
||||
//
|
||||
// PostgreSQL remains the single source of truth; this package keeps a
|
||||
// secondary, per-tenant Manticore Search index (Hybrid BM25+Vektor is a later
|
||||
// phase) in sync with the documents table. Only the write/sync half is
|
||||
// implemented here — there is deliberately NO search endpoint yet (Phase 2/3).
|
||||
//
|
||||
// Design guarantees:
|
||||
// - The index is best-effort. When Manticore is not configured (empty DSN)
|
||||
// the whole thing degrades to a no-op: the Indexer is nil and every caller
|
||||
// skips silently.
|
||||
// - An index error must NEVER be propagated to the originating HTTP request.
|
||||
// Callers log and move on. Postgres stays authoritative, so a stale index
|
||||
// is a recoverable, non-fatal condition (a later reindex CLI, Phase 3,
|
||||
// rebuilds it).
|
||||
//
|
||||
// This package intentionally has NO dependency on internal/storage to avoid an
|
||||
// import cycle: storage builds DocumentDoc values and calls into here.
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DocumentDoc is the index representation of a stored document. It is the
|
||||
// projection of a documents row plus its resolved taxonomy (tags) and ACL
|
||||
// (visibility group IDs) that the search index needs.
|
||||
type DocumentDoc struct {
|
||||
ID int64
|
||||
TenantID int64
|
||||
Title string
|
||||
DocType string // deprecated free-text doc_type
|
||||
Correspondent string // deprecated free-text correspondent
|
||||
OCRText string
|
||||
Tags []string
|
||||
TagIDs []int64
|
||||
DocTypeID *int64
|
||||
CorrespondentID *int64
|
||||
ACLGroupIDs []int64
|
||||
RetainUntil *time.Time
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
// SearchQuery is the (Phase 3) full-text + attribute query against a single
|
||||
// tenant's index. It intentionally carries only what the index needs to return
|
||||
// a ranked list of documents.id values; the caller re-hydrates the full
|
||||
// document rows from Postgres (the source of truth) afterwards.
|
||||
type SearchQuery struct {
|
||||
// Query is the raw user full-text term. It is escaped before it ever
|
||||
// reaches a MATCH() expression — callers pass it verbatim.
|
||||
Query string
|
||||
// TagIDs, when non-empty, restricts hits to documents carrying ANY of
|
||||
// these tag ids (MVA filter).
|
||||
TagIDs []int64
|
||||
// DocTypeID, when non-nil, restricts hits to that document type.
|
||||
DocTypeID *int64
|
||||
// ACLGroupIDs applies the group-resolved document ACL: when non-nil, only
|
||||
// documents visible to ANY of these permission groups are returned. A nil
|
||||
// slice means "no ACL filter" (domain_admin/superadmin). An explicitly
|
||||
// empty (non-nil) slice would match nothing — callers must short-circuit
|
||||
// that case before querying.
|
||||
ACLGroupIDs []int64
|
||||
// Page is 1-based; PageSize caps hits per page.
|
||||
Page int
|
||||
PageSize int
|
||||
}
|
||||
|
||||
// SearchHit is a single ranked result: a documents.id plus its BM25 score.
|
||||
type SearchHit struct {
|
||||
ID int64
|
||||
Score float64
|
||||
}
|
||||
|
||||
// Indexer syncs a single (tenant-scoped) document index. Implementations must
|
||||
// never block or fail the calling request on transient backend errors beyond
|
||||
// returning the error for the caller to log.
|
||||
type Indexer interface {
|
||||
// IndexSync inserts or replaces the document (id-based upsert).
|
||||
IndexSync(ctx context.Context, doc DocumentDoc) error
|
||||
// Delete removes the document from the index by its documents.id.
|
||||
Delete(ctx context.Context, id int64) error
|
||||
// Search runs a full-text + attribute query and returns the ranked hits
|
||||
// for the requested page plus the total match count (across all pages).
|
||||
Search(ctx context.Context, q SearchQuery) (hits []SearchHit, total int, err error)
|
||||
}
|
||||
|
||||
// TenantIndexer hands out per-tenant Indexer instances, each backed by its own
|
||||
// RT table (documents_tenant_<id>).
|
||||
type TenantIndexer interface {
|
||||
ForTenant(tenantID int64) Indexer
|
||||
Close() error
|
||||
}
|
||||
@@ -0,0 +1,319 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// validTableName guards against SQL injection through table-name
|
||||
// interpolation: only documents_tenant_<digits> is ever a legal RT table.
|
||||
var validTableName = regexp.MustCompile(`^documents_tenant_\d+$`)
|
||||
|
||||
// manticoreIndex implements Indexer against a single Manticore RT table.
|
||||
type manticoreIndex struct {
|
||||
db *sql.DB
|
||||
table string
|
||||
}
|
||||
|
||||
// ManticoreTenantManager implements TenantIndexer using Manticore Search via
|
||||
// the MySQL wire protocol (port 9306 by default). No CGO required — pure Go
|
||||
// through database/sql + github.com/go-sql-driver/mysql.
|
||||
type ManticoreTenantManager struct {
|
||||
db *sql.DB
|
||||
mu sync.RWMutex
|
||||
pool map[int64]*manticoreIndex
|
||||
}
|
||||
|
||||
// NewManticoreTenantManager opens (and pings) a Manticore connection and
|
||||
// returns a ready manager. Per-tenant RT tables are created lazily on first
|
||||
// ForTenant use.
|
||||
func NewManticoreTenantManager(dsn string) (*ManticoreTenantManager, error) {
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("manticore: open: %w", err)
|
||||
}
|
||||
db.SetMaxOpenConns(16)
|
||||
db.SetMaxIdleConns(4)
|
||||
db.SetConnMaxLifetime(5 * time.Minute)
|
||||
|
||||
if err := db.Ping(); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("manticore: ping: %w", err)
|
||||
}
|
||||
|
||||
return &ManticoreTenantManager{
|
||||
db: db,
|
||||
pool: make(map[int64]*manticoreIndex),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ForTenant returns the Indexer for a tenant, creating its RT table on first
|
||||
// use. If the table cannot be ensured, a no-op Indexer is returned so callers
|
||||
// never panic or block — the miss is the caller's to log.
|
||||
func (m *ManticoreTenantManager) ForTenant(tenantID int64) Indexer {
|
||||
if tenantID <= 0 {
|
||||
return noopIndexer{}
|
||||
}
|
||||
|
||||
m.mu.RLock()
|
||||
idx, ok := m.pool[tenantID]
|
||||
m.mu.RUnlock()
|
||||
if ok {
|
||||
return idx
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if idx, ok = m.pool[tenantID]; ok {
|
||||
return idx
|
||||
}
|
||||
|
||||
idx = &manticoreIndex{db: m.db, table: manticoreTableName(tenantID)}
|
||||
if err := idx.ensureTable(); err != nil {
|
||||
return noopIndexer{}
|
||||
}
|
||||
m.pool[tenantID] = idx
|
||||
return idx
|
||||
}
|
||||
|
||||
// Close closes the shared database connection.
|
||||
func (m *ManticoreTenantManager) Close() error {
|
||||
return m.db.Close()
|
||||
}
|
||||
|
||||
// ── manticoreIndex methods ────────────────────────────────────────────────
|
||||
|
||||
// ensureTable creates the RT index idempotently if it does not yet exist.
|
||||
func (idx *manticoreIndex) ensureTable() error {
|
||||
stmt := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
doc_id string,
|
||||
title text,
|
||||
doc_type text,
|
||||
correspondent text,
|
||||
ocr_text text,
|
||||
tags text,
|
||||
tag_ids multi,
|
||||
doc_type_id bigint,
|
||||
correspondent_id bigint,
|
||||
acl_group_ids multi,
|
||||
retain_until_ts bigint,
|
||||
created_ts bigint,
|
||||
updated_ts bigint,
|
||||
deleted uint
|
||||
) type='rt' morphology='lemmatize_de_all,stem_en'`, idx.table)
|
||||
if _, err := idx.db.Exec(stmt); err != nil {
|
||||
return fmt.Errorf("manticore: ensureTable %s: %w", idx.table, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IndexSync upserts a document via REPLACE INTO (id-based, Manticore-typical).
|
||||
//
|
||||
// The two MVA (multi) columns tag_ids/acl_group_ids are interpolated inline
|
||||
// because Manticore does not accept bind placeholders inside the (a,b,c) MVA
|
||||
// value syntax. This is injection-safe: both lists are rendered from int64
|
||||
// values only (joinInts), never from free text.
|
||||
func (idx *manticoreIndex) IndexSync(ctx context.Context, doc DocumentDoc) error {
|
||||
_, err := idx.db.ExecContext(ctx,
|
||||
fmt.Sprintf(`REPLACE INTO %s
|
||||
(id, doc_id, title, doc_type, correspondent, ocr_text, tags, tag_ids, doc_type_id, correspondent_id, acl_group_ids, retain_until_ts, created_ts, updated_ts, deleted)
|
||||
VALUES (?,?,?,?,?,?,?,(%s),?,?,(%s),?,?,?,?)`, idx.table, joinInts(doc.TagIDs), joinInts(doc.ACLGroupIDs)),
|
||||
doc.ID,
|
||||
fmt.Sprintf("%d", doc.ID),
|
||||
doc.Title,
|
||||
doc.DocType,
|
||||
doc.Correspondent,
|
||||
doc.OCRText,
|
||||
strings.Join(doc.Tags, " "),
|
||||
ptrInt64(doc.DocTypeID),
|
||||
ptrInt64(doc.CorrespondentID),
|
||||
unixOrZero(doc.RetainUntil),
|
||||
doc.CreatedAt.Unix(),
|
||||
doc.UpdatedAt.Unix(),
|
||||
0,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("manticore: IndexSync %s id=%d: %w", idx.table, doc.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes a document from the RT index by its documents.id.
|
||||
func (idx *manticoreIndex) Delete(ctx context.Context, id int64) error {
|
||||
_, err := idx.db.ExecContext(ctx,
|
||||
fmt.Sprintf("DELETE FROM %s WHERE id = ?", idx.table), id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("manticore: Delete %s id=%d: %w", idx.table, id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Search runs a full-text + attribute query against the RT index and returns
|
||||
// the ranked hits for the requested page plus the overall match count.
|
||||
//
|
||||
// The full-text term (if any) is matched against the title, ocr_text, tags,
|
||||
// correspondent and doc_type fields. It is escaped via escapeMatch before it
|
||||
// is placed into a MATCH() expression — the only user-controlled string in the
|
||||
// query; every other filter value is an int64 rendered inline (injection-safe)
|
||||
// or a bound placeholder.
|
||||
func (idx *manticoreIndex) Search(ctx context.Context, q SearchQuery) ([]SearchHit, int, error) {
|
||||
var whereParts []string
|
||||
var args []any
|
||||
|
||||
hasMatch := strings.TrimSpace(q.Query) != ""
|
||||
if hasMatch {
|
||||
whereParts = append(whereParts, "MATCH(?)")
|
||||
args = append(args, "@(title,ocr_text,tags,correspondent,doc_type) "+escapeMatch(q.Query))
|
||||
}
|
||||
|
||||
// Never return purged documents.
|
||||
whereParts = append(whereParts, "deleted = 0")
|
||||
|
||||
// Attribute filters. MVA lists are rendered inline from int64 values only
|
||||
// (joinInts) — Manticore rejects placeholders inside ANY(...) IN (...).
|
||||
if len(q.TagIDs) > 0 {
|
||||
whereParts = append(whereParts, fmt.Sprintf("ANY(tag_ids) IN (%s)", joinInts(q.TagIDs)))
|
||||
}
|
||||
if q.DocTypeID != nil {
|
||||
whereParts = append(whereParts, "doc_type_id = ?")
|
||||
args = append(args, *q.DocTypeID)
|
||||
}
|
||||
if q.ACLGroupIDs != nil {
|
||||
// A non-nil but empty slice means "no visible groups" — match nothing.
|
||||
if len(q.ACLGroupIDs) == 0 {
|
||||
return nil, 0, nil
|
||||
}
|
||||
whereParts = append(whereParts, fmt.Sprintf("ANY(acl_group_ids) IN (%s)", joinInts(q.ACLGroupIDs)))
|
||||
}
|
||||
|
||||
whereClause := ""
|
||||
if len(whereParts) > 0 {
|
||||
whereClause = "WHERE " + strings.Join(whereParts, " AND ")
|
||||
}
|
||||
|
||||
// Total match count (across all pages) for pagination metadata.
|
||||
countArgs := make([]any, len(args))
|
||||
copy(countArgs, args)
|
||||
countSQL := fmt.Sprintf("SELECT COUNT(*) FROM %s %s OPTION max_matches=1000000", idx.table, whereClause)
|
||||
var total int
|
||||
if err := idx.db.QueryRowContext(ctx, countSQL, countArgs...).Scan(&total); err != nil {
|
||||
return nil, 0, fmt.Errorf("manticore: Search count %s: %w", idx.table, err)
|
||||
}
|
||||
|
||||
pageSize := q.PageSize
|
||||
if pageSize <= 0 {
|
||||
pageSize = 20
|
||||
}
|
||||
page := q.Page
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
offset := (page - 1) * pageSize
|
||||
|
||||
scoreExpr := "1 as score"
|
||||
orderBy := "created_ts DESC"
|
||||
if hasMatch {
|
||||
scoreExpr = "WEIGHT() as score"
|
||||
orderBy = "WEIGHT() DESC, created_ts DESC"
|
||||
}
|
||||
|
||||
selectSQL := fmt.Sprintf(
|
||||
"SELECT id, %s FROM %s %s ORDER BY %s LIMIT ? OFFSET ? OPTION max_matches=10000",
|
||||
scoreExpr, idx.table, whereClause, orderBy)
|
||||
selectArgs := make([]any, len(args))
|
||||
copy(selectArgs, args)
|
||||
selectArgs = append(selectArgs, pageSize, offset)
|
||||
|
||||
rows, err := idx.db.QueryContext(ctx, selectSQL, selectArgs...)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("manticore: Search select %s: %w", idx.table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var hits []SearchHit
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var score float64
|
||||
if err := rows.Scan(&id, &score); err != nil {
|
||||
return nil, 0, fmt.Errorf("manticore: Search scan %s: %w", idx.table, err)
|
||||
}
|
||||
hits = append(hits, SearchHit{ID: id, Score: score})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, 0, fmt.Errorf("manticore: Search rows %s: %w", idx.table, err)
|
||||
}
|
||||
return hits, total, nil
|
||||
}
|
||||
|
||||
// escapeMatch escapes characters that carry special meaning in a Manticore
|
||||
// MATCH() expression, so a user-supplied full-text term can never inject
|
||||
// operators (query-injection guard). Mirrors the established archivmail
|
||||
// escapeManticoreMatch pattern.
|
||||
func escapeMatch(s string) string {
|
||||
const specials = `\()|!@~"/^$=<`
|
||||
var b strings.Builder
|
||||
b.Grow(len(s))
|
||||
for _, c := range s {
|
||||
if strings.ContainsRune(specials, c) {
|
||||
b.WriteRune('\\')
|
||||
}
|
||||
b.WriteRune(c)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// ── helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
// manticoreTableName returns the RT table name for a tenant. Panics on an
|
||||
// invalid result — that would be a programming error, not a runtime condition.
|
||||
func manticoreTableName(tenantID int64) string {
|
||||
name := fmt.Sprintf("documents_tenant_%d", tenantID)
|
||||
if !validTableName.MatchString(name) {
|
||||
panic(fmt.Sprintf("manticore: invalid table name: %q", name))
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// joinInts renders an int64 slice as a comma-separated list for a Manticore
|
||||
// multi (MVA) column value, wrapped in parentheses by the caller's placeholder.
|
||||
func joinInts(ids []int64) string {
|
||||
if len(ids) == 0 {
|
||||
return ""
|
||||
}
|
||||
parts := make([]string, len(ids))
|
||||
for i, v := range ids {
|
||||
parts[i] = fmt.Sprintf("%d", v)
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func ptrInt64(p *int64) int64 {
|
||||
if p == nil {
|
||||
return 0
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
func unixOrZero(t *time.Time) int64 {
|
||||
if t == nil || t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.Unix()
|
||||
}
|
||||
|
||||
// noopIndexer is returned when a tenant table cannot be ensured. Every method
|
||||
// silently succeeds so a backend hiccup never blocks the calling request.
|
||||
type noopIndexer struct{}
|
||||
|
||||
func (noopIndexer) IndexSync(context.Context, DocumentDoc) error { return nil }
|
||||
func (noopIndexer) Delete(context.Context, int64) error { return nil }
|
||||
func (noopIndexer) Search(context.Context, SearchQuery) ([]SearchHit, int, error) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
Reference in New Issue
Block a user