patrick
9a24ea29e1
Git-Repository für bestehenden archivdms-Code initialisiert, Branch-/Commit-Konvention (feature/<ticket>-<slug>-Branches, Ticket-Prefix in Commit-Nachricht) etabliert.
320 lines
9.7 KiB
Go
320 lines
9.7 KiB
Go
package index
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"regexp"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
_ "github.com/go-sql-driver/mysql"
|
|
)
|
|
|
|
// validTableName guards against SQL injection through table-name
|
|
// interpolation: only documents_tenant_<digits> is ever a legal RT table.
|
|
var validTableName = regexp.MustCompile(`^documents_tenant_\d+$`)
|
|
|
|
// manticoreIndex implements Indexer against a single Manticore RT table.
|
|
type manticoreIndex struct {
|
|
db *sql.DB
|
|
table string
|
|
}
|
|
|
|
// ManticoreTenantManager implements TenantIndexer using Manticore Search via
|
|
// the MySQL wire protocol (port 9306 by default). No CGO required — pure Go
|
|
// through database/sql + github.com/go-sql-driver/mysql.
|
|
type ManticoreTenantManager struct {
|
|
db *sql.DB
|
|
mu sync.RWMutex
|
|
pool map[int64]*manticoreIndex
|
|
}
|
|
|
|
// NewManticoreTenantManager opens (and pings) a Manticore connection and
|
|
// returns a ready manager. Per-tenant RT tables are created lazily on first
|
|
// ForTenant use.
|
|
func NewManticoreTenantManager(dsn string) (*ManticoreTenantManager, error) {
|
|
db, err := sql.Open("mysql", dsn)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("manticore: open: %w", err)
|
|
}
|
|
db.SetMaxOpenConns(16)
|
|
db.SetMaxIdleConns(4)
|
|
db.SetConnMaxLifetime(5 * time.Minute)
|
|
|
|
if err := db.Ping(); err != nil {
|
|
db.Close()
|
|
return nil, fmt.Errorf("manticore: ping: %w", err)
|
|
}
|
|
|
|
return &ManticoreTenantManager{
|
|
db: db,
|
|
pool: make(map[int64]*manticoreIndex),
|
|
}, nil
|
|
}
|
|
|
|
// ForTenant returns the Indexer for a tenant, creating its RT table on first
|
|
// use. If the table cannot be ensured, a no-op Indexer is returned so callers
|
|
// never panic or block — the miss is the caller's to log.
|
|
func (m *ManticoreTenantManager) ForTenant(tenantID int64) Indexer {
|
|
if tenantID <= 0 {
|
|
return noopIndexer{}
|
|
}
|
|
|
|
m.mu.RLock()
|
|
idx, ok := m.pool[tenantID]
|
|
m.mu.RUnlock()
|
|
if ok {
|
|
return idx
|
|
}
|
|
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
if idx, ok = m.pool[tenantID]; ok {
|
|
return idx
|
|
}
|
|
|
|
idx = &manticoreIndex{db: m.db, table: manticoreTableName(tenantID)}
|
|
if err := idx.ensureTable(); err != nil {
|
|
return noopIndexer{}
|
|
}
|
|
m.pool[tenantID] = idx
|
|
return idx
|
|
}
|
|
|
|
// Close closes the shared database connection.
|
|
func (m *ManticoreTenantManager) Close() error {
|
|
return m.db.Close()
|
|
}
|
|
|
|
// ── manticoreIndex methods ────────────────────────────────────────────────
|
|
|
|
// ensureTable creates the RT index idempotently if it does not yet exist.
|
|
func (idx *manticoreIndex) ensureTable() error {
|
|
stmt := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
|
doc_id string,
|
|
title text,
|
|
doc_type text,
|
|
correspondent text,
|
|
ocr_text text,
|
|
tags text,
|
|
tag_ids multi,
|
|
doc_type_id bigint,
|
|
correspondent_id bigint,
|
|
acl_group_ids multi,
|
|
retain_until_ts bigint,
|
|
created_ts bigint,
|
|
updated_ts bigint,
|
|
deleted uint
|
|
) type='rt' morphology='lemmatize_de_all,stem_en'`, idx.table)
|
|
if _, err := idx.db.Exec(stmt); err != nil {
|
|
return fmt.Errorf("manticore: ensureTable %s: %w", idx.table, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// IndexSync upserts a document via REPLACE INTO (id-based, Manticore-typical).
|
|
//
|
|
// The two MVA (multi) columns tag_ids/acl_group_ids are interpolated inline
|
|
// because Manticore does not accept bind placeholders inside the (a,b,c) MVA
|
|
// value syntax. This is injection-safe: both lists are rendered from int64
|
|
// values only (joinInts), never from free text.
|
|
func (idx *manticoreIndex) IndexSync(ctx context.Context, doc DocumentDoc) error {
|
|
_, err := idx.db.ExecContext(ctx,
|
|
fmt.Sprintf(`REPLACE INTO %s
|
|
(id, doc_id, title, doc_type, correspondent, ocr_text, tags, tag_ids, doc_type_id, correspondent_id, acl_group_ids, retain_until_ts, created_ts, updated_ts, deleted)
|
|
VALUES (?,?,?,?,?,?,?,(%s),?,?,(%s),?,?,?,?)`, idx.table, joinInts(doc.TagIDs), joinInts(doc.ACLGroupIDs)),
|
|
doc.ID,
|
|
fmt.Sprintf("%d", doc.ID),
|
|
doc.Title,
|
|
doc.DocType,
|
|
doc.Correspondent,
|
|
doc.OCRText,
|
|
strings.Join(doc.Tags, " "),
|
|
ptrInt64(doc.DocTypeID),
|
|
ptrInt64(doc.CorrespondentID),
|
|
unixOrZero(doc.RetainUntil),
|
|
doc.CreatedAt.Unix(),
|
|
doc.UpdatedAt.Unix(),
|
|
0,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("manticore: IndexSync %s id=%d: %w", idx.table, doc.ID, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Delete removes a document from the RT index by its documents.id.
|
|
func (idx *manticoreIndex) Delete(ctx context.Context, id int64) error {
|
|
_, err := idx.db.ExecContext(ctx,
|
|
fmt.Sprintf("DELETE FROM %s WHERE id = ?", idx.table), id)
|
|
if err != nil {
|
|
return fmt.Errorf("manticore: Delete %s id=%d: %w", idx.table, id, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Search runs a full-text + attribute query against the RT index and returns
|
|
// the ranked hits for the requested page plus the overall match count.
|
|
//
|
|
// The full-text term (if any) is matched against the title, ocr_text, tags,
|
|
// correspondent and doc_type fields. It is escaped via escapeMatch before it
|
|
// is placed into a MATCH() expression — the only user-controlled string in the
|
|
// query; every other filter value is an int64 rendered inline (injection-safe)
|
|
// or a bound placeholder.
|
|
func (idx *manticoreIndex) Search(ctx context.Context, q SearchQuery) ([]SearchHit, int, error) {
|
|
var whereParts []string
|
|
var args []any
|
|
|
|
hasMatch := strings.TrimSpace(q.Query) != ""
|
|
if hasMatch {
|
|
whereParts = append(whereParts, "MATCH(?)")
|
|
args = append(args, "@(title,ocr_text,tags,correspondent,doc_type) "+escapeMatch(q.Query))
|
|
}
|
|
|
|
// Never return purged documents.
|
|
whereParts = append(whereParts, "deleted = 0")
|
|
|
|
// Attribute filters. MVA lists are rendered inline from int64 values only
|
|
// (joinInts) — Manticore rejects placeholders inside ANY(...) IN (...).
|
|
if len(q.TagIDs) > 0 {
|
|
whereParts = append(whereParts, fmt.Sprintf("ANY(tag_ids) IN (%s)", joinInts(q.TagIDs)))
|
|
}
|
|
if q.DocTypeID != nil {
|
|
whereParts = append(whereParts, "doc_type_id = ?")
|
|
args = append(args, *q.DocTypeID)
|
|
}
|
|
if q.ACLGroupIDs != nil {
|
|
// A non-nil but empty slice means "no visible groups" — match nothing.
|
|
if len(q.ACLGroupIDs) == 0 {
|
|
return nil, 0, nil
|
|
}
|
|
whereParts = append(whereParts, fmt.Sprintf("ANY(acl_group_ids) IN (%s)", joinInts(q.ACLGroupIDs)))
|
|
}
|
|
|
|
whereClause := ""
|
|
if len(whereParts) > 0 {
|
|
whereClause = "WHERE " + strings.Join(whereParts, " AND ")
|
|
}
|
|
|
|
// Total match count (across all pages) for pagination metadata.
|
|
countArgs := make([]any, len(args))
|
|
copy(countArgs, args)
|
|
countSQL := fmt.Sprintf("SELECT COUNT(*) FROM %s %s OPTION max_matches=1000000", idx.table, whereClause)
|
|
var total int
|
|
if err := idx.db.QueryRowContext(ctx, countSQL, countArgs...).Scan(&total); err != nil {
|
|
return nil, 0, fmt.Errorf("manticore: Search count %s: %w", idx.table, err)
|
|
}
|
|
|
|
pageSize := q.PageSize
|
|
if pageSize <= 0 {
|
|
pageSize = 20
|
|
}
|
|
page := q.Page
|
|
if page <= 0 {
|
|
page = 1
|
|
}
|
|
offset := (page - 1) * pageSize
|
|
|
|
scoreExpr := "1 as score"
|
|
orderBy := "created_ts DESC"
|
|
if hasMatch {
|
|
scoreExpr = "WEIGHT() as score"
|
|
orderBy = "WEIGHT() DESC, created_ts DESC"
|
|
}
|
|
|
|
selectSQL := fmt.Sprintf(
|
|
"SELECT id, %s FROM %s %s ORDER BY %s LIMIT ? OFFSET ? OPTION max_matches=10000",
|
|
scoreExpr, idx.table, whereClause, orderBy)
|
|
selectArgs := make([]any, len(args))
|
|
copy(selectArgs, args)
|
|
selectArgs = append(selectArgs, pageSize, offset)
|
|
|
|
rows, err := idx.db.QueryContext(ctx, selectSQL, selectArgs...)
|
|
if err != nil {
|
|
return nil, 0, fmt.Errorf("manticore: Search select %s: %w", idx.table, err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
var hits []SearchHit
|
|
for rows.Next() {
|
|
var id int64
|
|
var score float64
|
|
if err := rows.Scan(&id, &score); err != nil {
|
|
return nil, 0, fmt.Errorf("manticore: Search scan %s: %w", idx.table, err)
|
|
}
|
|
hits = append(hits, SearchHit{ID: id, Score: score})
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, 0, fmt.Errorf("manticore: Search rows %s: %w", idx.table, err)
|
|
}
|
|
return hits, total, nil
|
|
}
|
|
|
|
// escapeMatch escapes characters that carry special meaning in a Manticore
|
|
// MATCH() expression, so a user-supplied full-text term can never inject
|
|
// operators (query-injection guard). Mirrors the established archivmail
|
|
// escapeManticoreMatch pattern.
|
|
func escapeMatch(s string) string {
|
|
const specials = `\()|!@~"/^$=<`
|
|
var b strings.Builder
|
|
b.Grow(len(s))
|
|
for _, c := range s {
|
|
if strings.ContainsRune(specials, c) {
|
|
b.WriteRune('\\')
|
|
}
|
|
b.WriteRune(c)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// ── helpers ────────────────────────────────────────────────────────────────
|
|
|
|
// manticoreTableName returns the RT table name for a tenant. Panics on an
|
|
// invalid result — that would be a programming error, not a runtime condition.
|
|
func manticoreTableName(tenantID int64) string {
|
|
name := fmt.Sprintf("documents_tenant_%d", tenantID)
|
|
if !validTableName.MatchString(name) {
|
|
panic(fmt.Sprintf("manticore: invalid table name: %q", name))
|
|
}
|
|
return name
|
|
}
|
|
|
|
// joinInts renders an int64 slice as a comma-separated list for a Manticore
|
|
// multi (MVA) column value, wrapped in parentheses by the caller's placeholder.
|
|
func joinInts(ids []int64) string {
|
|
if len(ids) == 0 {
|
|
return ""
|
|
}
|
|
parts := make([]string, len(ids))
|
|
for i, v := range ids {
|
|
parts[i] = fmt.Sprintf("%d", v)
|
|
}
|
|
return strings.Join(parts, ",")
|
|
}
|
|
|
|
func ptrInt64(p *int64) int64 {
|
|
if p == nil {
|
|
return 0
|
|
}
|
|
return *p
|
|
}
|
|
|
|
func unixOrZero(t *time.Time) int64 {
|
|
if t == nil || t.IsZero() {
|
|
return 0
|
|
}
|
|
return t.Unix()
|
|
}
|
|
|
|
// noopIndexer is returned when a tenant table cannot be ensured. Every method
|
|
// silently succeeds so a backend hiccup never blocks the calling request.
|
|
type noopIndexer struct{}
|
|
|
|
func (noopIndexer) IndexSync(context.Context, DocumentDoc) error { return nil }
|
|
func (noopIndexer) Delete(context.Context, int64) error { return nil }
|
|
func (noopIndexer) Search(context.Context, SearchQuery) ([]SearchHit, int, error) {
|
|
return nil, 0, nil
|
|
}
|