FDN-01: repository & projektgerüst
Git-Repository für bestehenden archivdms-Code initialisiert, Branch-/Commit-Konvention (feature/<ticket>-<slug>-Branches, Ticket-Prefix in Commit-Nachricht) etabliert.
This commit is contained in:
@@ -0,0 +1,173 @@
|
||||
// Per-tenant API keys for the read-only Buchhaltungs-Pull-API (see
|
||||
// migrations/026_accounting_api_keys.sql and
|
||||
// internal/api/accounting_handlers.go).
|
||||
//
|
||||
// Token handling follows exactly the share-link pattern in shares.go: the raw
|
||||
// key is generated once (32 bytes crypto/rand, base64url, with a fixed
|
||||
// "adms_" prefix so it is recognisable in logs/config files), returned to the
|
||||
// caller EXACTLY once at creation time, and only its hex SHA-256 hash is ever
|
||||
// persisted. Lookup for authentication is always by key_hash, never by id.
|
||||
//
|
||||
// Keys are never hard-deleted: revoking only sets revoked_at, so the audit
|
||||
// trail of which key pulled which documents stays resolvable (GoBD
|
||||
// Nachvollziehbarkeit).
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// accountingKeyPrefix marks a raw accounting API key as such. It is part of
|
||||
// the hashed value (the whole string is hashed), it is NOT a separate column.
|
||||
const accountingKeyPrefix = "adms_"
|
||||
|
||||
// ErrAccountingKeyNotFound is returned when a key lookup (by id+tenant or by
|
||||
// key_hash) matches no usable row — unknown key, wrong tenant, or revoked.
|
||||
var ErrAccountingKeyNotFound = errors.New("storage: accounting api key not found")
|
||||
|
||||
// AccountingAPIKey is the safe view of an accounting_api_keys row. The hash is
|
||||
// deliberately NOT part of this struct so it can never be serialised into an
|
||||
// API response, and the plaintext key exists only as the second return value
|
||||
// of CreateAccountingAPIKey.
|
||||
type AccountingAPIKey struct {
|
||||
ID int64 `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
Label string `json:"label"`
|
||||
CreatedBy *int64 `json:"created_by,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
RevokedAt *time.Time `json:"revoked_at,omitempty"`
|
||||
LastUsedAt *time.Time `json:"last_used_at,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Store) initAccountingAPIKeysSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
-- No FK on tenant_id / created_by: consistent with the rest of the
|
||||
-- schema (plain BIGINT), because tenants/users are owned by other
|
||||
-- stores that initialise after storage.New().
|
||||
CREATE TABLE IF NOT EXISTS accounting_api_keys (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
key_hash TEXT NOT NULL UNIQUE,
|
||||
label TEXT NOT NULL DEFAULT '',
|
||||
created_by BIGINT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
revoked_at TIMESTAMPTZ,
|
||||
last_used_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_accounting_api_keys_tenant ON accounting_api_keys(tenant_id);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create accounting_api_keys table: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// hashAccountingKey returns the hex-encoded SHA-256 of a raw accounting API
|
||||
// key, the value persisted in / looked up from accounting_api_keys.key_hash.
|
||||
func hashAccountingKey(key string) string {
|
||||
sum := sha256.Sum256([]byte(key))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// CreateAccountingAPIKey inserts a new key for a tenant and returns the stored
|
||||
// row plus the raw (plaintext) key. The plaintext is returned ONLY here and
|
||||
// never again — only its SHA-256 hash is persisted.
|
||||
func (s *Store) CreateAccountingAPIKey(ctx context.Context, tenantID int64, label string, createdBy *int64) (*AccountingAPIKey, string, error) {
|
||||
rawBytes := make([]byte, 32)
|
||||
if _, err := rand.Read(rawBytes); err != nil {
|
||||
return nil, "", fmt.Errorf("storage: generate accounting api key: %w", err)
|
||||
}
|
||||
key := accountingKeyPrefix + base64.RawURLEncoding.EncodeToString(rawBytes)
|
||||
|
||||
var k AccountingAPIKey
|
||||
err := s.db.QueryRow(ctx, `
|
||||
INSERT INTO accounting_api_keys (tenant_id, key_hash, label, created_by)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
RETURNING id, tenant_id, label, created_by, created_at, revoked_at, last_used_at
|
||||
`, tenantID, hashAccountingKey(key), label, createdBy,
|
||||
).Scan(&k.ID, &k.TenantID, &k.Label, &k.CreatedBy, &k.CreatedAt, &k.RevokedAt, &k.LastUsedAt)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("storage: create accounting api key: %w", err)
|
||||
}
|
||||
return &k, key, nil
|
||||
}
|
||||
|
||||
// ResolveAccountingAPIKey authenticates a raw key: it hashes the key, looks the
|
||||
// row up by key_hash, rejects revoked keys, refreshes last_used_at and returns
|
||||
// the owning tenant id plus the key id.
|
||||
//
|
||||
// The returned tenantID is THE ONLY trusted tenant source for the pull
|
||||
// endpoints — no caller may take a tenant_id from the request itself.
|
||||
// Unknown and revoked keys both yield ErrAccountingKeyNotFound so the caller
|
||||
// cannot distinguish them.
|
||||
func (s *Store) ResolveAccountingAPIKey(ctx context.Context, rawKey string) (tenantID int64, keyID int64, err error) {
|
||||
if rawKey == "" {
|
||||
return 0, 0, ErrAccountingKeyNotFound
|
||||
}
|
||||
// Single statement: authenticate + touch last_used_at atomically. The
|
||||
// revoked_at IS NULL guard lives in the WHERE clause, so a revoked key can
|
||||
// never return a tenant id.
|
||||
err = s.db.QueryRow(ctx, `
|
||||
UPDATE accounting_api_keys
|
||||
SET last_used_at = now()
|
||||
WHERE key_hash = $1 AND revoked_at IS NULL
|
||||
RETURNING tenant_id, id
|
||||
`, hashAccountingKey(rawKey)).Scan(&tenantID, &keyID)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return 0, 0, ErrAccountingKeyNotFound
|
||||
}
|
||||
return 0, 0, fmt.Errorf("storage: resolve accounting api key: %w", err)
|
||||
}
|
||||
return tenantID, keyID, nil
|
||||
}
|
||||
|
||||
// ListAccountingAPIKeys returns all keys of a tenant (including revoked ones —
|
||||
// no hard delete), newest first. Never returns the hash or the plaintext.
|
||||
func (s *Store) ListAccountingAPIKeys(ctx context.Context, tenantID int64) ([]AccountingAPIKey, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, tenant_id, label, created_by, created_at, revoked_at, last_used_at
|
||||
FROM accounting_api_keys
|
||||
WHERE tenant_id = $1
|
||||
ORDER BY created_at DESC
|
||||
`, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list accounting api keys: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]AccountingAPIKey, 0)
|
||||
for rows.Next() {
|
||||
var k AccountingAPIKey
|
||||
if err := rows.Scan(&k.ID, &k.TenantID, &k.Label, &k.CreatedBy, &k.CreatedAt, &k.RevokedAt, &k.LastUsedAt); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan accounting api key: %w", err)
|
||||
}
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RevokeAccountingAPIKey marks a key as revoked (never hard-deleted), scoped to
|
||||
// tenant ownership (IDOR guard: id AND tenant_id). Returns
|
||||
// ErrAccountingKeyNotFound when no key of that id belongs to the tenant.
|
||||
func (s *Store) RevokeAccountingAPIKey(ctx context.Context, id, tenantID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE accounting_api_keys SET revoked_at = now()
|
||||
WHERE id = $1 AND tenant_id = $2 AND revoked_at IS NULL
|
||||
`, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: revoke accounting api key: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrAccountingKeyNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
// Read-only query layer for the Buchhaltungs-Pull-API (see
|
||||
// internal/api/accounting_handlers.go). Deliberately separate from
|
||||
// documents.go's ListDocuments: the accounting view is a machine-to-machine
|
||||
// export with its own reduced projection (no ocr_text, no storage_path, no
|
||||
// content_hash) and keyset pagination over (created_at, id).
|
||||
//
|
||||
// Tenant isolation: every query here takes tenantID as its FIRST parameter and
|
||||
// filters `WHERE d.tenant_id = $1` — there is no variant without it. The
|
||||
// caller (the Bearer-auth middleware) derives that id solely from the resolved
|
||||
// API key, never from the request.
|
||||
//
|
||||
// No permission-group ACL filter is applied: an accounting API key is a
|
||||
// tenant-level machine credential (like the SFTP inbox account), not a user
|
||||
// session. That is why creating one requires domain_admin.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// ErrInvalidAccountingCursor is returned when a client-supplied cursor cannot
|
||||
// be decoded. The handler maps this to HTTP 400.
|
||||
var ErrInvalidAccountingCursor = errors.New("storage: invalid accounting cursor")
|
||||
|
||||
// AccountingDocument is the reduced, export-safe projection of a document for
|
||||
// the pull API. storage_path / content_hash / ocr_text are intentionally
|
||||
// absent (GoBD/security: the WORM location is never exposed; the file is only
|
||||
// reachable through the streaming endpoint).
|
||||
type AccountingDocument struct {
|
||||
ID int64 `json:"id"`
|
||||
Title string `json:"title"`
|
||||
DocumentDate *time.Time `json:"document_date,omitempty"`
|
||||
DocumentDateScore *float64 `json:"document_date_score,omitempty"`
|
||||
DocTypeID *int64 `json:"doc_type_id,omitempty"`
|
||||
DocType string `json:"doc_type,omitempty"`
|
||||
CorrespondentID *int64 `json:"correspondent_id,omitempty"`
|
||||
Correspondent string `json:"correspondent,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// AccountingDocumentFilter holds the (already validated) query parameters of
|
||||
// GET /api/v1/accounting/documents. TenantID is NOT part of it on purpose — it
|
||||
// is passed separately from the API-key context so it can never be overwritten
|
||||
// by a decoded request body/query.
|
||||
type AccountingDocumentFilter struct {
|
||||
// Since/Until bound document_date (inclusive/exclusive respectively).
|
||||
Since *time.Time
|
||||
Until *time.Time
|
||||
// DocTypeID restricts to one document type.
|
||||
DocTypeID *int64
|
||||
// MinDateScore is the confidence quality gate (e.g. 0.75); documents with a
|
||||
// NULL score are excluded as soon as this is set.
|
||||
MinDateScore *float64
|
||||
// Cursor is the opaque keyset cursor from a previous page ("" = first page).
|
||||
Cursor string
|
||||
// Limit is the page size (already clamped by the handler).
|
||||
Limit int
|
||||
}
|
||||
|
||||
// AccountingPage is one page of pull results plus the cursor for the next one.
|
||||
type AccountingPage struct {
|
||||
Documents []AccountingDocument `json:"documents"`
|
||||
NextCursor string `json:"next_cursor,omitempty"`
|
||||
HasMore bool `json:"has_more"`
|
||||
}
|
||||
|
||||
// encodeAccountingCursor builds the opaque keyset cursor from the last row of a
|
||||
// page. Format (base64url of) "<unix_nanos>:<id>" — the exact tuple the ORDER
|
||||
// BY / WHERE comparison uses.
|
||||
func encodeAccountingCursor(createdAt time.Time, id int64) string {
|
||||
raw := strconv.FormatInt(createdAt.UTC().UnixNano(), 10) + ":" + strconv.FormatInt(id, 10)
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(raw))
|
||||
}
|
||||
|
||||
// decodeAccountingCursor parses a cursor produced by encodeAccountingCursor.
|
||||
func decodeAccountingCursor(cursor string) (time.Time, int64, error) {
|
||||
b, err := base64.RawURLEncoding.DecodeString(cursor)
|
||||
if err != nil {
|
||||
return time.Time{}, 0, ErrInvalidAccountingCursor
|
||||
}
|
||||
parts := strings.SplitN(string(b), ":", 2)
|
||||
if len(parts) != 2 {
|
||||
return time.Time{}, 0, ErrInvalidAccountingCursor
|
||||
}
|
||||
nanos, err := strconv.ParseInt(parts[0], 10, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, 0, ErrInvalidAccountingCursor
|
||||
}
|
||||
id, err := strconv.ParseInt(parts[1], 10, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, 0, ErrInvalidAccountingCursor
|
||||
}
|
||||
return time.Unix(0, nanos).UTC(), id, nil
|
||||
}
|
||||
|
||||
// ListAccountingDocuments returns one keyset-paginated page of a tenant's
|
||||
// documents for the pull API, ordered by (created_at, id) ascending so a
|
||||
// consumer can poll incrementally without ever re-reading or skipping rows.
|
||||
// Soft-deleted (trashed) documents are excluded.
|
||||
func (s *Store) ListAccountingDocuments(ctx context.Context, tenantID int64, f AccountingDocumentFilter) (*AccountingPage, error) {
|
||||
limit := f.Limit
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
|
||||
// $1 is always the tenant id — the isolation predicate is not optional.
|
||||
args := []any{tenantID}
|
||||
where := []string{"d.tenant_id = $1", "d.deleted_at IS NULL"}
|
||||
|
||||
if f.Since != nil {
|
||||
args = append(args, *f.Since)
|
||||
where = append(where, fmt.Sprintf("d.document_date >= $%d", len(args)))
|
||||
}
|
||||
if f.Until != nil {
|
||||
args = append(args, *f.Until)
|
||||
where = append(where, fmt.Sprintf("d.document_date < $%d", len(args)))
|
||||
}
|
||||
if f.DocTypeID != nil {
|
||||
args = append(args, *f.DocTypeID)
|
||||
where = append(where, fmt.Sprintf("d.doc_type_id = $%d", len(args)))
|
||||
}
|
||||
if f.MinDateScore != nil {
|
||||
args = append(args, *f.MinDateScore)
|
||||
where = append(where, fmt.Sprintf("d.document_date_score IS NOT NULL AND d.document_date_score >= $%d", len(args)))
|
||||
}
|
||||
if f.Cursor != "" {
|
||||
curTS, curID, err := decodeAccountingCursor(f.Cursor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
args = append(args, curTS, curID)
|
||||
where = append(where, fmt.Sprintf("(d.created_at, d.id) > ($%d, $%d)", len(args)-1, len(args)))
|
||||
}
|
||||
|
||||
// Fetch one extra row to detect whether a further page exists.
|
||||
args = append(args, limit+1)
|
||||
query := `
|
||||
SELECT d.id, d.title, d.document_date, d.document_date_score,
|
||||
d.doc_type_id, COALESCE(dt.name, COALESCE(d.doc_type, '')),
|
||||
d.correspondent_id, COALESCE(c.name, COALESCE(d.correspondent, '')),
|
||||
d.created_at
|
||||
FROM documents d
|
||||
LEFT JOIN document_types dt ON dt.id = d.doc_type_id AND dt.tenant_id = d.tenant_id
|
||||
LEFT JOIN correspondents c ON c.id = d.correspondent_id AND c.tenant_id = d.tenant_id
|
||||
WHERE ` + strings.Join(where, " AND ") + `
|
||||
ORDER BY d.created_at ASC, d.id ASC
|
||||
LIMIT $` + strconv.Itoa(len(args))
|
||||
|
||||
rows, err := s.db.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list accounting documents: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]AccountingDocument, 0, limit)
|
||||
for rows.Next() {
|
||||
var d AccountingDocument
|
||||
if err := rows.Scan(&d.ID, &d.Title, &d.DocumentDate, &d.DocumentDateScore,
|
||||
&d.DocTypeID, &d.DocType, &d.CorrespondentID, &d.Correspondent, &d.CreatedAt); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan accounting document: %w", err)
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("storage: list accounting documents: %w", err)
|
||||
}
|
||||
|
||||
page := &AccountingPage{Documents: out}
|
||||
if len(out) > limit {
|
||||
page.Documents = out[:limit]
|
||||
page.HasMore = true
|
||||
last := page.Documents[limit-1]
|
||||
page.NextCursor = encodeAccountingCursor(last.CreatedAt, last.ID)
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// AccountingFileRef carries the server-side-only information needed to stream a
|
||||
// document's WORM file. The storage path is unexported and reachable only via
|
||||
// StoragePath(), mirroring storage.ResolvedShare, so a handler cannot
|
||||
// accidentally serialise it into a response.
|
||||
type AccountingFileRef struct {
|
||||
DocumentID int64
|
||||
Title string
|
||||
storagePath string
|
||||
}
|
||||
|
||||
// StoragePath returns the WORM path of the file (server-side only).
|
||||
func (r *AccountingFileRef) StoragePath() string { return r.storagePath }
|
||||
|
||||
// GetAccountingDocumentFile resolves a document's WORM file location, scoped to
|
||||
// the tenant of the API key (id AND tenant_id — IDOR guard). Returns
|
||||
// ErrDocumentNotFound for unknown id, foreign tenant and trashed document
|
||||
// alike, so the endpoint never reveals whether a document exists outside the
|
||||
// caller's tenant.
|
||||
func (s *Store) GetAccountingDocumentFile(ctx context.Context, id, tenantID int64) (*AccountingFileRef, error) {
|
||||
var ref AccountingFileRef
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT id, title, storage_path
|
||||
FROM documents
|
||||
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
|
||||
`, id, tenantID).Scan(&ref.DocumentID, &ref.Title, &ref.storagePath)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrDocumentNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("storage: get accounting document file: %w", err)
|
||||
}
|
||||
return &ref, nil
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// ErrAkteNotFound is returned when a tenant-scoped akte lookup/update affects
|
||||
// zero rows (wrong id or wrong tenant).
|
||||
var ErrAkteNotFound = errors.New("storage: akte not found")
|
||||
|
||||
// Akte is a digital file folder ("digitaler Aktenordner") grouping documents
|
||||
// in a strict 1:n relationship via documents.akte_id. It has NO own ACL — an
|
||||
// akte's visibility is derived from the documents it contains (see
|
||||
// ListAkteDocuments, which applies the exact same document_visibility EXISTS
|
||||
// clause as ListDocuments). See project_akte_konzept_plan.md.
|
||||
type Akte struct {
|
||||
ID int64 `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
Titel string `json:"titel"`
|
||||
Beschreibung string `json:"beschreibung"`
|
||||
Status string `json:"status"`
|
||||
CorrespondentID *int64 `json:"correspondent_id,omitempty"`
|
||||
CreatedBy *int64 `json:"created_by,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ClosedAt *time.Time `json:"closed_at,omitempty"`
|
||||
// DocumentCount is a computed field (COUNT of documents with this akte_id),
|
||||
// not a stored column. Populated by ListAkten/GetAkte for the list view.
|
||||
DocumentCount int `json:"document_count"`
|
||||
}
|
||||
|
||||
// initAktenSchema creates the akten table and adds the documents.akte_id FK
|
||||
// column. Order matters: the akten table must exist BEFORE the ALTER TABLE on
|
||||
// documents because akte_id references akten(id). Idempotent. Wired into
|
||||
// (*Store).initSchema (documents.go) after the other schema hooks.
|
||||
func (s *Store) initAktenSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS akten (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
titel TEXT NOT NULL,
|
||||
beschreibung TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'offen' CHECK (status IN ('offen','geschlossen')),
|
||||
correspondent_id BIGINT REFERENCES correspondents(id),
|
||||
created_by BIGINT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
closed_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_akten_tenant ON akten (tenant_id);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create akten table: %w", err)
|
||||
}
|
||||
|
||||
// documents.akte_id: strict 1:n container link. ON DELETE SET NULL is the
|
||||
// GoBD safeguard — deleting an akte can NEVER delete documents, it only
|
||||
// decouples them (see project_akte_konzept_plan.md).
|
||||
_, err = s.db.Exec(ctx, `
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS akte_id BIGINT REFERENCES akten(id) ON DELETE SET NULL;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: alter documents add akte_id: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateAkte inserts a new akte and returns it (document_count is 0 for a
|
||||
// freshly created akte).
|
||||
func (s *Store) CreateAkte(ctx context.Context, tenantID int64, titel, beschreibung string, correspondentID *int64, createdBy int64) (*Akte, error) {
|
||||
var a Akte
|
||||
err := s.db.QueryRow(ctx, `
|
||||
INSERT INTO akten (tenant_id, titel, beschreibung, correspondent_id, created_by)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING id, tenant_id, titel, beschreibung, status, correspondent_id, created_by, created_at, closed_at
|
||||
`, tenantID, titel, beschreibung, correspondentID, createdBy).Scan(
|
||||
&a.ID, &a.TenantID, &a.Titel, &a.Beschreibung, &a.Status, &a.CorrespondentID, &a.CreatedBy, &a.CreatedAt, &a.ClosedAt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: create akte: %w", err)
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// ListAkten returns all akten for a tenant, newest first, each with its
|
||||
// document_count. No ACL filter here: an akte itself is metadata; its contained
|
||||
// documents are ACL-filtered at ListAkteDocuments time.
|
||||
func (s *Store) ListAkten(ctx context.Context, tenantID int64) ([]Akte, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT a.id, a.tenant_id, a.titel, a.beschreibung, a.status, a.correspondent_id, a.created_by, a.created_at, a.closed_at,
|
||||
(SELECT COUNT(*) FROM documents d WHERE d.akte_id = a.id AND d.deleted_at IS NULL) AS document_count
|
||||
FROM akten a
|
||||
WHERE a.tenant_id = $1
|
||||
ORDER BY a.created_at DESC
|
||||
`, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list akten: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]Akte, 0)
|
||||
for rows.Next() {
|
||||
var a Akte
|
||||
if err := rows.Scan(&a.ID, &a.TenantID, &a.Titel, &a.Beschreibung, &a.Status, &a.CorrespondentID, &a.CreatedBy, &a.CreatedAt, &a.ClosedAt, &a.DocumentCount); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan akte: %w", err)
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// GetAkte retrieves a single akte by id, scoped to tenant, with document_count.
|
||||
// Returns ErrAkteNotFound when no row matches.
|
||||
func (s *Store) GetAkte(ctx context.Context, id, tenantID int64) (*Akte, error) {
|
||||
var a Akte
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT a.id, a.tenant_id, a.titel, a.beschreibung, a.status, a.correspondent_id, a.created_by, a.created_at, a.closed_at,
|
||||
(SELECT COUNT(*) FROM documents d WHERE d.akte_id = a.id AND d.deleted_at IS NULL) AS document_count
|
||||
FROM akten a
|
||||
WHERE a.id = $1 AND a.tenant_id = $2
|
||||
`, id, tenantID).Scan(&a.ID, &a.TenantID, &a.Titel, &a.Beschreibung, &a.Status, &a.CorrespondentID, &a.CreatedBy, &a.CreatedAt, &a.ClosedAt, &a.DocumentCount)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrAkteNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("storage: get akte: %w", err)
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// UpdateAkte changes an akte's titel/beschreibung/correspondent, scoped to
|
||||
// tenant. Returns ErrAkteNotFound when no row matches.
|
||||
func (s *Store) UpdateAkte(ctx context.Context, id, tenantID int64, titel, beschreibung string, correspondentID *int64) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE akten SET titel = $1, beschreibung = $2, correspondent_id = $3
|
||||
WHERE id = $4 AND tenant_id = $5
|
||||
`, titel, beschreibung, correspondentID, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: update akte: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrAkteNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CloseAkte marks an akte as geschlossen and stamps closed_at, scoped to
|
||||
// tenant. Returns ErrAkteNotFound when no row matches.
|
||||
func (s *Store) CloseAkte(ctx context.Context, id, tenantID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE akten SET status = 'geschlossen', closed_at = now()
|
||||
WHERE id = $1 AND tenant_id = $2
|
||||
`, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: close akte: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrAkteNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteAkte hard-deletes an akte, scoped to tenant. Documents are NOT deleted:
|
||||
// the ON DELETE SET NULL FK on documents.akte_id automatically decouples them.
|
||||
// Returns ErrAkteNotFound when no row matches.
|
||||
func (s *Store) DeleteAkte(ctx context.Context, id, tenantID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `DELETE FROM akten WHERE id = $1 AND tenant_id = $2`, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: delete akte: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrAkteNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListAkteDocuments returns all documents assigned to an akte, ACL-filtered
|
||||
// exactly like ListDocuments (aclUserID non-nil for role 'user', nil for
|
||||
// domain_admin/superadmin). Same document_visibility EXISTS clause plus the
|
||||
// created_by ownership fallback.
|
||||
func (s *Store) ListAkteDocuments(ctx context.Context, akteID, tenantID int64, aclUserID *int64) ([]Document, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, tenant_id, title, COALESCE(doc_type, ''), COALESCE(correspondent, ''), doc_type_id, correspondent_id, storage_path, content_hash,
|
||||
COALESCE(ocr_text, ''), retain_until, COALESCE(source, ''), COALESCE(source_ref, ''), created_by, title_manually_set, created_at, updated_at
|
||||
FROM documents
|
||||
WHERE tenant_id = $1 AND akte_id = $2 AND deleted_at IS NULL
|
||||
AND ($3::bigint IS NULL OR documents.created_by = $3 OR EXISTS (
|
||||
SELECT 1 FROM document_visibility dv
|
||||
JOIN permission_group_members pgm ON pgm.group_id = dv.group_id
|
||||
WHERE dv.document_id = documents.id AND pgm.user_id = $3
|
||||
))
|
||||
ORDER BY created_at DESC
|
||||
`, tenantID, akteID, aclUserID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list akte documents: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]Document, 0)
|
||||
for rows.Next() {
|
||||
var d Document
|
||||
if err := rows.Scan(&d.ID, &d.TenantID, &d.Title, &d.DocType, &d.Correspondent, &d.DocTypeID, &d.CorrespondentID, &d.StoragePath, &d.ContentHash,
|
||||
&d.OCRText, &d.RetainUntil, &d.Source, &d.SourceRef, &d.CreatedBy, &d.TitleManuallySet, &d.CreatedAt, &d.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan akte document: %w", err)
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SetDocumentAkte assigns (akteID non-nil) or removes (akteID nil) a document's
|
||||
// akte membership, scoped to tenant. Analogous to SetDocumentDocType, but the
|
||||
// akte is not part of the document ACL, so no visibility recompute is needed —
|
||||
// only a search-index re-sync. Returns ErrDocumentNotFound when no row matches.
|
||||
func (s *Store) SetDocumentAkte(ctx context.Context, documentID, tenantID int64, akteID *int64) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE documents SET akte_id = $1, updated_at = now()
|
||||
WHERE id = $2 AND tenant_id = $3
|
||||
`, akteID, documentID, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: set document akte: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrDocumentNotFound
|
||||
}
|
||||
s.SyncIndex(ctx, documentID)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,453 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
// ErrClassificationTemplateNotFound is returned when a classification-template
|
||||
// lookup, update or delete does not match any row owned by the caller's tenant.
|
||||
var ErrClassificationTemplateNotFound = errors.New("storage: classification template not found or not owned by tenant")
|
||||
|
||||
// ErrDuplicateTemplateName is returned when a tenant already has a
|
||||
// classification template with the same name (UNIQUE(tenant_id, name)).
|
||||
var ErrDuplicateTemplateName = errors.New("storage: classification template with this name already exists for tenant")
|
||||
|
||||
// ClassificationTemplate is a tenant-scoped "Klassifizierungsvorlage": a named
|
||||
// bundle of a document type, tags, custom-field default values and a retention
|
||||
// period that can be applied to a document in one action. Deliberately NOT
|
||||
// persistently coupled to any document (no template_id column on documents) so
|
||||
// a later template edit can never retroactively change past documents — the
|
||||
// application is recorded only via an audit-log entry (GoBD-Nachvollziehbarkeit).
|
||||
type ClassificationTemplate struct {
|
||||
ID int64 `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description,omitempty"`
|
||||
DocTypeID *int64 `json:"doc_type_id,omitempty"`
|
||||
RetainYears *int `json:"retain_years,omitempty"`
|
||||
Active bool `json:"active"`
|
||||
CreatedBy *int64 `json:"created_by,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
Tags []TaxonomyEntity `json:"tags"`
|
||||
FieldDefaults []TemplateFieldDefault `json:"field_defaults"`
|
||||
|
||||
// TitleTemplate is an optional Go text/template pattern used to derive the
|
||||
// document title when this template is applied (see
|
||||
// classification_templates_title.go). NULL/empty means "no template title"
|
||||
// — the tenant-wide default_title_template is tried next, and if that is
|
||||
// also empty the document's existing title is kept.
|
||||
TitleTemplate *string `json:"title_template,omitempty"`
|
||||
}
|
||||
|
||||
// TemplateFieldDefault is one resolved custom-field default of a template,
|
||||
// joined to its field definition. Exactly one value column is expected to be
|
||||
// populated (matching the field's type). overwrite carries the template's
|
||||
// intent to overwrite an already-set document value (only honoured on an
|
||||
// explicit confirmed apply — see ApplyTemplate).
|
||||
type TemplateFieldDefault struct {
|
||||
FieldID int64 `json:"field_id"`
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
FieldType string `json:"field_type"`
|
||||
ValueText *string `json:"value_text,omitempty"`
|
||||
ValueNumber *float64 `json:"value_number,omitempty"`
|
||||
ValueDate *time.Time `json:"value_date,omitempty"`
|
||||
ValueBool *bool `json:"value_bool,omitempty"`
|
||||
Overwrite bool `json:"overwrite"`
|
||||
}
|
||||
|
||||
// TemplateFieldDefaultInput is one supplied default in a bulk PUT. Exactly one
|
||||
// of the value pointers is expected to be populated (matching the field type).
|
||||
type TemplateFieldDefaultInput struct {
|
||||
FieldID int64 `json:"field_id"`
|
||||
ValueText *string `json:"value_text,omitempty"`
|
||||
ValueNumber *float64 `json:"value_number,omitempty"`
|
||||
ValueDate *string `json:"value_date,omitempty"` // ISO date "2006-01-02"
|
||||
ValueBool *bool `json:"value_bool,omitempty"`
|
||||
Overwrite bool `json:"overwrite"`
|
||||
}
|
||||
|
||||
// CreateTemplateRequest holds create parameters for a classification template.
|
||||
type CreateTemplateRequest struct {
|
||||
Name string
|
||||
Description string
|
||||
DocTypeID *int64
|
||||
RetainYears *int
|
||||
Active bool
|
||||
CreatedBy *int64
|
||||
TitleTemplate *string
|
||||
}
|
||||
|
||||
// UpdateTemplateRequest holds update parameters for a classification template.
|
||||
type UpdateTemplateRequest struct {
|
||||
Name string
|
||||
Description string
|
||||
DocTypeID *int64
|
||||
RetainYears *int
|
||||
Active bool
|
||||
TitleTemplate *string
|
||||
}
|
||||
|
||||
// initClassificationTemplatesSchema creates the classification_templates /
|
||||
// classification_template_tags / classification_template_field_defaults tables.
|
||||
// Idempotent, called from (*Store).initSchema AFTER initTaxonomySchema and
|
||||
// initCustomFieldsSchema (FK dependency on document_types / custom_field_defs).
|
||||
// Documented (not executed) in migrations/011_classification_templates.sql.
|
||||
func (s *Store) initClassificationTemplatesSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS classification_templates (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
doc_type_id BIGINT REFERENCES document_types(id) ON DELETE SET NULL,
|
||||
retain_years INT,
|
||||
active BOOLEAN NOT NULL DEFAULT true,
|
||||
created_by BIGINT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, name)
|
||||
);
|
||||
ALTER TABLE classification_templates ADD COLUMN IF NOT EXISTS title_template TEXT;
|
||||
CREATE INDEX IF NOT EXISTS idx_classification_templates_tenant ON classification_templates(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_classification_templates_doc_type ON classification_templates(doc_type_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS classification_template_tags (
|
||||
template_id BIGINT NOT NULL REFERENCES classification_templates(id) ON DELETE CASCADE,
|
||||
tag_id BIGINT NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (template_id, tag_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS classification_template_field_defaults (
|
||||
template_id BIGINT NOT NULL REFERENCES classification_templates(id) ON DELETE CASCADE,
|
||||
field_id BIGINT NOT NULL REFERENCES custom_field_defs(id) ON DELETE CASCADE,
|
||||
value_text TEXT,
|
||||
value_number NUMERIC,
|
||||
value_date DATE,
|
||||
value_bool BOOLEAN,
|
||||
overwrite BOOLEAN NOT NULL DEFAULT false,
|
||||
PRIMARY KEY (template_id, field_id)
|
||||
);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create classification templates tables: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func scanClassificationTemplate(row interface {
|
||||
Scan(dest ...any) error
|
||||
}) (*ClassificationTemplate, error) {
|
||||
var t ClassificationTemplate
|
||||
if err := row.Scan(&t.ID, &t.TenantID, &t.Name, &t.Description, &t.DocTypeID, &t.RetainYears,
|
||||
&t.Active, &t.CreatedBy, &t.CreatedAt, &t.UpdatedAt, &t.TitleTemplate); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.Tags = make([]TaxonomyEntity, 0)
|
||||
t.FieldDefaults = make([]TemplateFieldDefault, 0)
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
const classificationTemplateCols = `id, tenant_id, name, COALESCE(description, ''), doc_type_id, retain_years, active, created_by, created_at, updated_at, title_template`
|
||||
|
||||
// CreateTemplate inserts a new classification template (without tags / field
|
||||
// defaults — those are set via SetTemplateTags / SetTemplateFieldDefaults).
|
||||
func (s *Store) CreateTemplate(ctx context.Context, tenantID int64, req CreateTemplateRequest) (*ClassificationTemplate, error) {
|
||||
row := s.db.QueryRow(ctx, `
|
||||
INSERT INTO classification_templates (tenant_id, name, description, doc_type_id, retain_years, active, created_by, title_template)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
RETURNING `+classificationTemplateCols,
|
||||
tenantID, req.Name, nullIfEmpty(req.Description), req.DocTypeID, req.RetainYears, req.Active, req.CreatedBy, nullIfEmptyPtr(req.TitleTemplate))
|
||||
t, err := scanClassificationTemplate(row)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return nil, ErrDuplicateTemplateName
|
||||
}
|
||||
return nil, fmt.Errorf("storage: create classification template: %w", err)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// ListTemplates returns all classification templates for a tenant, optionally
|
||||
// filtered by document type. Returns a non-nil (possibly empty) slice.
|
||||
func (s *Store) ListTemplates(ctx context.Context, tenantID int64, docTypeID *int64) ([]ClassificationTemplate, error) {
|
||||
query := `SELECT ` + classificationTemplateCols + ` FROM classification_templates WHERE tenant_id = $1`
|
||||
args := []any{tenantID}
|
||||
if docTypeID != nil {
|
||||
query += ` AND doc_type_id = $2`
|
||||
args = append(args, *docTypeID)
|
||||
}
|
||||
query += ` ORDER BY name ASC`
|
||||
rows, err := s.db.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list classification templates: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]ClassificationTemplate, 0)
|
||||
for rows.Next() {
|
||||
t, err := scanClassificationTemplate(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: scan classification template: %w", err)
|
||||
}
|
||||
out = append(out, *t)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// GetTemplate returns one classification template resolved with its tags and
|
||||
// custom-field defaults, scoped to tenant ownership.
|
||||
func (s *Store) GetTemplate(ctx context.Context, id, tenantID int64) (*ClassificationTemplate, error) {
|
||||
row := s.db.QueryRow(ctx, `SELECT `+classificationTemplateCols+`
|
||||
FROM classification_templates WHERE id = $1 AND tenant_id = $2`, id, tenantID)
|
||||
t, err := scanClassificationTemplate(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrClassificationTemplateNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("storage: get classification template: %w", err)
|
||||
}
|
||||
|
||||
tags, err := s.listTemplateTags(ctx, id, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.Tags = tags
|
||||
|
||||
defaults, err := s.listTemplateFieldDefaults(ctx, id, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.FieldDefaults = defaults
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// listTemplateTags returns the tags attached to a template, scoped to tenant.
|
||||
func (s *Store) listTemplateTags(ctx context.Context, templateID, tenantID int64) ([]TaxonomyEntity, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT t.id, t.tenant_id, t.name, COALESCE(t.color, ''), t.match_algorithm, COALESCE(t.match_pattern, ''), t.case_sensitive, COALESCE(t.barcode_value, ''), t.created_at
|
||||
FROM tags t
|
||||
JOIN classification_template_tags ctt ON ctt.tag_id = t.id
|
||||
WHERE ctt.template_id = $1 AND t.tenant_id = $2
|
||||
ORDER BY t.name ASC
|
||||
`, templateID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list template tags: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]TaxonomyEntity, 0)
|
||||
for rows.Next() {
|
||||
e, err := scanTaxonomyEntity(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: scan template tag: %w", err)
|
||||
}
|
||||
out = append(out, *e)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// listTemplateFieldDefaults returns the custom-field defaults of a template,
|
||||
// joined to their definitions, scoped to tenant.
|
||||
func (s *Store) listTemplateFieldDefaults(ctx context.Context, templateID, tenantID int64) ([]TemplateFieldDefault, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT d.field_id, f.name, f.label, f.field_type,
|
||||
d.value_text, d.value_number, d.value_date, d.value_bool, d.overwrite
|
||||
FROM classification_template_field_defaults d
|
||||
JOIN custom_field_defs f ON f.id = d.field_id
|
||||
WHERE d.template_id = $1 AND f.tenant_id = $2
|
||||
ORDER BY f.name ASC
|
||||
`, templateID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list template field defaults: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]TemplateFieldDefault, 0)
|
||||
for rows.Next() {
|
||||
var d TemplateFieldDefault
|
||||
if err := rows.Scan(&d.FieldID, &d.Name, &d.Label, &d.FieldType,
|
||||
&d.ValueText, &d.ValueNumber, &d.ValueDate, &d.ValueBool, &d.Overwrite); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan template field default: %w", err)
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// UpdateTemplate updates a classification template's core attributes (not its
|
||||
// tags / field defaults), scoped to tenant ownership.
|
||||
func (s *Store) UpdateTemplate(ctx context.Context, id, tenantID int64, req UpdateTemplateRequest) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE classification_templates
|
||||
SET name = $1, description = $2, doc_type_id = $3, retain_years = $4, active = $5, title_template = $6, updated_at = now()
|
||||
WHERE id = $7 AND tenant_id = $8
|
||||
`, req.Name, nullIfEmpty(req.Description), req.DocTypeID, req.RetainYears, req.Active, nullIfEmptyPtr(req.TitleTemplate), id, tenantID)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return ErrDuplicateTemplateName
|
||||
}
|
||||
return fmt.Errorf("storage: update classification template: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrClassificationTemplateNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteTemplate deletes a classification template (cascades to its tags /
|
||||
// field defaults), scoped to tenant ownership.
|
||||
func (s *Store) DeleteTemplate(ctx context.Context, id, tenantID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `DELETE FROM classification_templates WHERE id = $1 AND tenant_id = $2`, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: delete classification template: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrClassificationTemplateNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// templateOwned returns true if the template belongs to the tenant.
|
||||
func (s *Store) templateOwned(ctx context.Context, templateID, tenantID int64) (bool, error) {
|
||||
var ok bool
|
||||
err := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM classification_templates WHERE id = $1 AND tenant_id = $2)`, templateID, tenantID).Scan(&ok)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("storage: check template ownership: %w", err)
|
||||
}
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
// SetTemplateTags replaces the complete set of tags on a template (bulk PUT).
|
||||
// All referenced tags must belong to the tenant. Scoped to tenant ownership of
|
||||
// the template. Delete-all + insert in one transaction (SetDocumentTypeFields
|
||||
// pattern).
|
||||
func (s *Store) SetTemplateTags(ctx context.Context, templateID, tenantID int64, tagIDs []int64) error {
|
||||
owns, err := s.templateOwned(ctx, templateID, tenantID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !owns {
|
||||
return ErrClassificationTemplateNotFound
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: begin set template tags: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM classification_template_tags WHERE template_id = $1`, templateID); err != nil {
|
||||
return fmt.Errorf("storage: clear template tags: %w", err)
|
||||
}
|
||||
for _, tagID := range tagIDs {
|
||||
var ok bool
|
||||
if err := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM tags WHERE id = $1 AND tenant_id = $2)`, tagID, tenantID).Scan(&ok); err != nil {
|
||||
return fmt.Errorf("storage: check tag ownership: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: tag_id %d", ErrTaxonomyNotFound, tagID)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO classification_template_tags (template_id, tag_id) VALUES ($1, $2)
|
||||
ON CONFLICT (template_id, tag_id) DO NOTHING
|
||||
`, templateID, tagID); err != nil {
|
||||
return fmt.Errorf("storage: insert template tag: %w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("storage: commit set template tags: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetTemplateFieldDefaults replaces the complete set of custom-field defaults
|
||||
// on a template (bulk PUT). All referenced fields must belong to the tenant and
|
||||
// their supplied value is validated against the field type. Scoped to tenant
|
||||
// ownership of the template. Delete-all + insert in one transaction.
|
||||
func (s *Store) SetTemplateFieldDefaults(ctx context.Context, templateID, tenantID int64, defaults []TemplateFieldDefaultInput) error {
|
||||
owns, err := s.templateOwned(ctx, templateID, tenantID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !owns {
|
||||
return ErrClassificationTemplateNotFound
|
||||
}
|
||||
|
||||
// Load field definitions for type resolution / validation.
|
||||
defs, err := s.ListCustomFieldDefs(ctx, tenantID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defByID := make(map[int64]CustomFieldDef, len(defs))
|
||||
for _, d := range defs {
|
||||
defByID[d.ID] = d
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: begin set template field defaults: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM classification_template_field_defaults WHERE template_id = $1`, templateID); err != nil {
|
||||
return fmt.Errorf("storage: clear template field defaults: %w", err)
|
||||
}
|
||||
for _, in := range defaults {
|
||||
def, ok := defByID[in.FieldID]
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: field_id %d", ErrCustomFieldNotFound, in.FieldID)
|
||||
}
|
||||
var (
|
||||
text *string
|
||||
number *float64
|
||||
date *time.Time
|
||||
bl *bool
|
||||
)
|
||||
switch def.FieldType {
|
||||
case "text":
|
||||
text = in.ValueText
|
||||
case "enum":
|
||||
if in.ValueText != nil && *in.ValueText != "" {
|
||||
if len(def.EnumOptions) > 0 && !containsString(def.EnumOptions, *in.ValueText) {
|
||||
return fmt.Errorf("storage: value %q not in enum options for field %q", *in.ValueText, def.Name)
|
||||
}
|
||||
}
|
||||
text = in.ValueText
|
||||
case "number", "monetary":
|
||||
number = in.ValueNumber
|
||||
case "date":
|
||||
if in.ValueDate != nil && *in.ValueDate != "" {
|
||||
parsed, err := time.Parse("2006-01-02", *in.ValueDate)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: invalid date %q for field %q: %w", *in.ValueDate, def.Name, err)
|
||||
}
|
||||
date = &parsed
|
||||
}
|
||||
case "boolean":
|
||||
bl = in.ValueBool
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO classification_template_field_defaults (template_id, field_id, value_text, value_number, value_date, value_bool, overwrite)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (template_id, field_id) DO UPDATE
|
||||
SET value_text = EXCLUDED.value_text, value_number = EXCLUDED.value_number,
|
||||
value_date = EXCLUDED.value_date, value_bool = EXCLUDED.value_bool, overwrite = EXCLUDED.overwrite
|
||||
`, templateID, in.FieldID, text, number, date, bl, in.Overwrite); err != nil {
|
||||
return fmt.Errorf("storage: insert template field default: %w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("storage: commit set template field defaults: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// ApplyTemplateResult is the outcome (or dry-run preview) of applying a
|
||||
// classification template to a document. It is produced identically by the
|
||||
// preview and the commit path so there is no logic duplication between them.
|
||||
type ApplyTemplateResult struct {
|
||||
TemplateID int64 `json:"template_id"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
TagsToAdd []TaxonomyEntity `json:"tags_to_add"`
|
||||
TagsAlreadySet []TaxonomyEntity `json:"tags_already_set"`
|
||||
FieldsToSet []FieldDefaultChange `json:"fields_to_set"`
|
||||
FieldsSkipped []FieldDefaultChange `json:"fields_skipped"`
|
||||
FieldsOverwritten []FieldDefaultChange `json:"fields_overwritten"`
|
||||
RetainUntilBefore *time.Time `json:"retain_until_before,omitempty"`
|
||||
RetainUntilAfter *time.Time `json:"retain_until_after,omitempty"`
|
||||
RetainUntilBlocked bool `json:"retain_until_blocked"`
|
||||
Applied bool `json:"applied"`
|
||||
}
|
||||
|
||||
// FieldDefaultChange describes a single custom-field default's effect on a
|
||||
// document (used in the to-set / skipped / overwritten buckets).
|
||||
type FieldDefaultChange struct {
|
||||
FieldID int64 `json:"field_id"`
|
||||
Name string `json:"name"`
|
||||
OldValue string `json:"old_value,omitempty"`
|
||||
NewValue string `json:"new_value"`
|
||||
}
|
||||
|
||||
// computeRetainUntil returns createdAt + retainYears years, or nil if
|
||||
// retainYears is nil (template does not set a retention period).
|
||||
func computeRetainUntil(createdAt time.Time, retainYears *int) *time.Time {
|
||||
if retainYears == nil {
|
||||
return nil
|
||||
}
|
||||
t := createdAt.AddDate(*retainYears, 0, 0)
|
||||
return &t
|
||||
}
|
||||
|
||||
// formatTemplateDefault renders a template field default's value as a string
|
||||
// for FieldDefaultChange.NewValue.
|
||||
func formatTemplateDefault(d TemplateFieldDefault) string {
|
||||
return formatFieldValueParts(d.ValueText, d.ValueNumber, d.ValueDate, d.ValueBool)
|
||||
}
|
||||
|
||||
// formatExistingValue renders a document's current field value as a string for
|
||||
// FieldDefaultChange.OldValue.
|
||||
func formatExistingValue(v DocumentFieldValue) string {
|
||||
return formatFieldValueParts(v.ValueText, v.ValueNumber, v.ValueDate, v.ValueBool)
|
||||
}
|
||||
|
||||
func formatFieldValueParts(text *string, number *float64, date *time.Time, bl *bool) string {
|
||||
switch {
|
||||
case text != nil:
|
||||
return *text
|
||||
case number != nil:
|
||||
return strconv.FormatFloat(*number, 'f', -1, 64)
|
||||
case date != nil:
|
||||
return date.Format("2006-01-02")
|
||||
case bl != nil:
|
||||
return strconv.FormatBool(*bl)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// PreviewApplyTemplate builds the full ApplyTemplateResult WITHOUT writing
|
||||
// anything. It is both the dry-run response and the basis the commit path
|
||||
// (ApplyTemplate) reuses. Field-default classification is done per-field on the
|
||||
// template default's own `overwrite` flag: an already-set value goes to
|
||||
// FieldsOverwritten when overwrite=true, otherwise to FieldsSkipped. The commit
|
||||
// path additionally requires an explicit confirm before actually overwriting.
|
||||
func (s *Store) PreviewApplyTemplate(ctx context.Context, documentID, templateID, tenantID int64) (*ApplyTemplateResult, error) {
|
||||
// GetDocument wraps a missing/foreign-tenant row as pgx.ErrNoRows (it does
|
||||
// not map to a sentinel itself) — translate it to ErrDocumentNotFound so the
|
||||
// handler can return a clean 404.
|
||||
doc, err := s.GetDocument(ctx, documentID, tenantID)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrDocumentNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tmpl, err := s.GetTemplate(ctx, templateID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := &ApplyTemplateResult{
|
||||
TemplateID: templateID,
|
||||
DryRun: true,
|
||||
TagsToAdd: make([]TaxonomyEntity, 0),
|
||||
TagsAlreadySet: make([]TaxonomyEntity, 0),
|
||||
FieldsToSet: make([]FieldDefaultChange, 0),
|
||||
FieldsSkipped: make([]FieldDefaultChange, 0),
|
||||
FieldsOverwritten: make([]FieldDefaultChange, 0),
|
||||
RetainUntilBefore: doc.RetainUntil,
|
||||
}
|
||||
|
||||
// --- tags ---
|
||||
existingTags, err := s.ListDocumentTags(ctx, documentID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
haveTag := make(map[int64]bool, len(existingTags))
|
||||
for _, t := range existingTags {
|
||||
haveTag[t.ID] = true
|
||||
}
|
||||
for _, t := range tmpl.Tags {
|
||||
if haveTag[t.ID] {
|
||||
res.TagsAlreadySet = append(res.TagsAlreadySet, t)
|
||||
} else {
|
||||
res.TagsToAdd = append(res.TagsToAdd, t)
|
||||
}
|
||||
}
|
||||
|
||||
// --- custom-field defaults ---
|
||||
existingVals, err := s.ListDocumentFieldValues(ctx, documentID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
existingByID := make(map[int64]DocumentFieldValue, len(existingVals))
|
||||
for _, v := range existingVals {
|
||||
existingByID[v.FieldID] = v
|
||||
}
|
||||
for _, d := range tmpl.FieldDefaults {
|
||||
change := FieldDefaultChange{FieldID: d.FieldID, Name: d.Name, NewValue: formatTemplateDefault(d)}
|
||||
existing, has := existingByID[d.FieldID]
|
||||
existingEmpty := !has || isEmptyResolved(existing.ValueText, existing.ValueNumber, existing.ValueDate, existing.ValueBool)
|
||||
if existingEmpty {
|
||||
res.FieldsToSet = append(res.FieldsToSet, change)
|
||||
continue
|
||||
}
|
||||
change.OldValue = formatExistingValue(existing)
|
||||
if d.Overwrite {
|
||||
res.FieldsOverwritten = append(res.FieldsOverwritten, change)
|
||||
} else {
|
||||
res.FieldsSkipped = append(res.FieldsSkipped, change)
|
||||
}
|
||||
}
|
||||
|
||||
// --- retention (absolute rule: never shorten, not even with overwrite) ---
|
||||
proposed := computeRetainUntil(doc.CreatedAt, tmpl.RetainYears)
|
||||
if doc.RetainUntil != nil && (proposed == nil || proposed.Before(*doc.RetainUntil)) {
|
||||
// A template must never shorten an existing retain_until.
|
||||
res.RetainUntilBlocked = true
|
||||
} else if proposed != nil {
|
||||
res.RetainUntilAfter = proposed
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// ApplyTemplate applies a classification template to a document. It first
|
||||
// builds the plan via PreviewApplyTemplate, then commits it: attaches the
|
||||
// missing tags, merges the field defaults into the document's existing values
|
||||
// (never a full replace), and extends retain_until — but ONLY when it is not
|
||||
// blocked (retention is never shortened, even with overwrite=true). The
|
||||
// per-field overwrite of an already-set value additionally requires the global
|
||||
// overwrite/confirm flag; otherwise those fields are demoted to FieldsSkipped.
|
||||
// The returned result reflects what was actually done (Applied=true).
|
||||
func (s *Store) ApplyTemplate(ctx context.Context, documentID, templateID, tenantID int64, overwrite bool) (*ApplyTemplateResult, error) {
|
||||
res, err := s.PreviewApplyTemplate(ctx, documentID, templateID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.DryRun = false
|
||||
|
||||
// Without an explicit confirm, overwriting an already-set field is not
|
||||
// performed — demote those to skipped so the caller sees they were kept.
|
||||
if !overwrite && len(res.FieldsOverwritten) > 0 {
|
||||
res.FieldsSkipped = append(res.FieldsSkipped, res.FieldsOverwritten...)
|
||||
res.FieldsOverwritten = res.FieldsOverwritten[:0]
|
||||
}
|
||||
|
||||
// --- attach missing tags (AttachTag runs RecomputeVisibility itself) ---
|
||||
for _, t := range res.TagsToAdd {
|
||||
if err := s.AttachTag(ctx, documentID, t.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// --- field defaults: merge with existing values, then set once ---
|
||||
// The set of field_ids the template actually writes (to-set + confirmed
|
||||
// overwrites). SetDocumentFieldValues is full-replace, so we start from the
|
||||
// document's current values and overlay only the template's writes.
|
||||
writeFields := make(map[int64]bool)
|
||||
for _, c := range res.FieldsToSet {
|
||||
writeFields[c.FieldID] = true
|
||||
}
|
||||
for _, c := range res.FieldsOverwritten {
|
||||
writeFields[c.FieldID] = true
|
||||
}
|
||||
if len(writeFields) > 0 {
|
||||
if err := s.applyTemplateFieldValues(ctx, documentID, templateID, tenantID, writeFields); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// --- retention: extend only, never shorten ---
|
||||
if !res.RetainUntilBlocked && res.RetainUntilAfter != nil {
|
||||
if _, err := s.db.Exec(ctx, `UPDATE documents SET retain_until = $1, updated_at = now() WHERE id = $2 AND tenant_id = $3`,
|
||||
*res.RetainUntilAfter, documentID, tenantID); err != nil {
|
||||
return nil, fmt.Errorf("storage: apply template retain_until: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// --- title: template-own title_template, else tenant default, else keep.
|
||||
// Runs AFTER tags/fields so tag-based patterns see the freshly attached
|
||||
// tags. Only touches non-manually-renamed documents; keeps
|
||||
// title_manually_set = false. Identical for the manual endpoint and the
|
||||
// workflow trigger (both call ApplyTemplate). ---
|
||||
tmpl, err := s.GetTemplate(ctx, templateID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.applyTemplateTitle(ctx, documentID, tenantID, tmpl); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res.Applied = true
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// applyTemplateFieldValues merges the template's writeFields defaults into the
|
||||
// document's current custom-field values and persists the union via
|
||||
// SetDocumentFieldValues (which is full-replace, hence the merge).
|
||||
func (s *Store) applyTemplateFieldValues(ctx context.Context, documentID, templateID, tenantID int64, writeFields map[int64]bool) error {
|
||||
existing, err := s.ListDocumentFieldValues(ctx, documentID, tenantID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defaults, err := s.listTemplateFieldDefaults(ctx, templateID, tenantID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defByID := make(map[int64]TemplateFieldDefault, len(defaults))
|
||||
for _, d := range defaults {
|
||||
defByID[d.FieldID] = d
|
||||
}
|
||||
|
||||
inputs := make([]DocumentFieldValueInput, 0, len(existing)+len(writeFields))
|
||||
seen := make(map[int64]bool)
|
||||
// Keep existing values, overlaying template writes where applicable.
|
||||
for _, v := range existing {
|
||||
seen[v.FieldID] = true
|
||||
if writeFields[v.FieldID] {
|
||||
if d, ok := defByID[v.FieldID]; ok {
|
||||
inputs = append(inputs, templateDefaultToInput(d))
|
||||
continue
|
||||
}
|
||||
}
|
||||
inputs = append(inputs, existingValueToInput(v))
|
||||
}
|
||||
// Template writes for fields the document did not have yet.
|
||||
for fid := range writeFields {
|
||||
if seen[fid] {
|
||||
continue
|
||||
}
|
||||
if d, ok := defByID[fid]; ok {
|
||||
inputs = append(inputs, templateDefaultToInput(d))
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := s.SetDocumentFieldValues(ctx, documentID, tenantID, inputs); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func templateDefaultToInput(d TemplateFieldDefault) DocumentFieldValueInput {
|
||||
in := DocumentFieldValueInput{
|
||||
FieldID: d.FieldID,
|
||||
ValueText: d.ValueText,
|
||||
ValueNumber: d.ValueNumber,
|
||||
ValueBool: d.ValueBool,
|
||||
}
|
||||
if d.ValueDate != nil {
|
||||
s := d.ValueDate.Format("2006-01-02")
|
||||
in.ValueDate = &s
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
func existingValueToInput(v DocumentFieldValue) DocumentFieldValueInput {
|
||||
in := DocumentFieldValueInput{
|
||||
FieldID: v.FieldID,
|
||||
ValueText: v.ValueText,
|
||||
ValueNumber: v.ValueNumber,
|
||||
ValueBool: v.ValueBool,
|
||||
}
|
||||
if v.ValueDate != nil {
|
||||
s := v.ValueDate.Format("2006-01-02")
|
||||
in.ValueDate = &s
|
||||
}
|
||||
return in
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// titleTemplateData is the data model exposed to a classification template's
|
||||
// (or the tenant-wide default's) title Go text/template. Fields are kept simple
|
||||
// (plain strings / times) so patterns stay readable, e.g.
|
||||
//
|
||||
// {{.Correspondent}} {{.DocumentType}} {{dateFormat "02.01.2006" .Belegdatum}}
|
||||
//
|
||||
// Belegdatum and UploadDate are passed as time.Time (zero value when unknown);
|
||||
// use the dateFormat template func to render them, which yields "" for a zero
|
||||
// time instead of Go's "0001-01-01..." default.
|
||||
type titleTemplateData struct {
|
||||
Correspondent string
|
||||
DocumentType string
|
||||
Belegdatum time.Time
|
||||
UploadDate time.Time
|
||||
Tags string
|
||||
OCRTitle string
|
||||
}
|
||||
|
||||
// titleTemplateFuncs provides the custom template functions available inside a
|
||||
// title template. dateFormat takes a Go reference layout ("02.01.2006") and a
|
||||
// time.Time, returning "" for a zero time so an unknown Belegdatum does not
|
||||
// leak a placeholder date into the title.
|
||||
var titleTemplateFuncs = template.FuncMap{
|
||||
"dateFormat": func(layout string, t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return t.Format(layout)
|
||||
},
|
||||
}
|
||||
|
||||
// ValidateTitleTemplate parses (but does not execute) a title template pattern
|
||||
// so the API layer can reject a syntactically invalid pattern up front. An
|
||||
// empty pattern is valid (means "no template title"). Exported for the
|
||||
// settings / template CRUD handlers.
|
||||
func ValidateTitleTemplate(pattern string) error {
|
||||
if strings.TrimSpace(pattern) == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := template.New("title").Option("missingkey=zero").Funcs(titleTemplateFuncs).Parse(pattern)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid title template: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// renderTitleTemplate parses and executes a title template against data. The
|
||||
// result is whitespace-trimmed. missingkey=zero guards against crashes when a
|
||||
// pattern references a field that does not exist. A parse/execute error or an
|
||||
// empty result is signalled to the caller so it can fall back (never an empty
|
||||
// title).
|
||||
func renderTitleTemplate(pattern string, data titleTemplateData) (string, error) {
|
||||
tmpl, err := template.New("title").Option("missingkey=zero").Funcs(titleTemplateFuncs).Parse(pattern)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("storage: parse title template: %w", err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := tmpl.Execute(&buf, data); err != nil {
|
||||
return "", fmt.Errorf("storage: execute title template: %w", err)
|
||||
}
|
||||
return strings.TrimSpace(buf.String()), nil
|
||||
}
|
||||
|
||||
// tenantDefaultTitleTemplate reads the tenant-wide fallback title template
|
||||
// straight from the tenants table (same DB pool). Returns "" when unset
|
||||
// (NULL) so callers can treat "no default" uniformly.
|
||||
func (s *Store) tenantDefaultTitleTemplate(ctx context.Context, tenantID int64) (string, error) {
|
||||
var v *string
|
||||
err := s.db.QueryRow(ctx, `SELECT default_title_template FROM tenants WHERE id = $1`, tenantID).Scan(&v)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", nil
|
||||
}
|
||||
return "", fmt.Errorf("storage: read tenant default_title_template: %w", err)
|
||||
}
|
||||
if v == nil {
|
||||
return "", nil
|
||||
}
|
||||
return strings.TrimSpace(*v), nil
|
||||
}
|
||||
|
||||
// taxonomyNameByID resolves a taxonomy entity's display name (tenant-scoped).
|
||||
// table must be a fixed internal literal ("correspondents" / "document_types"),
|
||||
// never user input. Returns "" (not an error) when the row does not exist so a
|
||||
// dangling reference cannot break title generation.
|
||||
func (s *Store) taxonomyNameByID(ctx context.Context, table string, id, tenantID int64) (string, error) {
|
||||
var name string
|
||||
err := s.db.QueryRow(ctx, fmt.Sprintf(`SELECT name FROM %s WHERE id = $1 AND tenant_id = $2`, table), id, tenantID).Scan(&name)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", nil
|
||||
}
|
||||
return "", fmt.Errorf("storage: lookup %s name: %w", table, err)
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
|
||||
// applyTemplateTitle derives and persists a document title from a classification
|
||||
// template's title_template (or, if that is empty, the tenant-wide
|
||||
// default_title_template). Rules (see feature spec):
|
||||
//
|
||||
// - Never touches a manually renamed document (title_manually_set = true).
|
||||
// - Chooses the template's own pattern first, then the tenant default; if both
|
||||
// are empty, does nothing (existing title kept).
|
||||
// - On an empty or errored render result, falls back to keeping the existing
|
||||
// title — never sets an empty string.
|
||||
// - Persists via UpdateDocumentTitleAuto so title_manually_set stays false
|
||||
// (a later correction + re-apply must still work).
|
||||
//
|
||||
// Called at the end of ApplyTemplate, so it runs identically for the manual
|
||||
// endpoint and the workflow trigger (both funnel through ApplyTemplate).
|
||||
func (s *Store) applyTemplateTitle(ctx context.Context, documentID, tenantID int64, tmpl *ClassificationTemplate) error {
|
||||
doc, err := s.GetDocument(ctx, documentID, tenantID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if doc.TitleManuallySet {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Resolve the effective pattern: template-own first, then tenant default.
|
||||
pattern := ""
|
||||
if tmpl.TitleTemplate != nil {
|
||||
pattern = strings.TrimSpace(*tmpl.TitleTemplate)
|
||||
}
|
||||
if pattern == "" {
|
||||
def, err := s.tenantDefaultTitleTemplate(ctx, tenantID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pattern = def
|
||||
}
|
||||
if pattern == "" {
|
||||
return nil // no template title configured at either level
|
||||
}
|
||||
|
||||
// Build the render data. Name lookups are best-effort (missing rows -> "").
|
||||
data := titleTemplateData{
|
||||
OCRTitle: doc.Title,
|
||||
UploadDate: doc.CreatedAt,
|
||||
}
|
||||
if doc.DocumentDate != nil {
|
||||
data.Belegdatum = *doc.DocumentDate
|
||||
}
|
||||
if doc.CorrespondentID != nil {
|
||||
name, err := s.taxonomyNameByID(ctx, "correspondents", *doc.CorrespondentID, tenantID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data.Correspondent = name
|
||||
}
|
||||
// Document type: prefer the template's target type (what is being applied),
|
||||
// falling back to the document's current type.
|
||||
docTypeID := tmpl.DocTypeID
|
||||
if docTypeID == nil {
|
||||
docTypeID = doc.DocTypeID
|
||||
}
|
||||
if docTypeID != nil {
|
||||
name, err := s.taxonomyNameByID(ctx, "document_types", *docTypeID, tenantID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data.DocumentType = name
|
||||
}
|
||||
tags, err := s.ListDocumentTags(ctx, documentID, tenantID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := make([]string, 0, len(tags))
|
||||
for _, t := range tags {
|
||||
names = append(names, t.Name)
|
||||
}
|
||||
data.Tags = strings.Join(names, ", ")
|
||||
|
||||
rendered, err := renderTitleTemplate(pattern, data)
|
||||
if err != nil || rendered == "" {
|
||||
// Fallback: keep the existing (OCR-derived) title, never blank it.
|
||||
return nil //nolint:nilerr // intentional: a bad template must not fail the apply
|
||||
}
|
||||
if rendered == doc.Title {
|
||||
return nil
|
||||
}
|
||||
return s.UpdateDocumentTitleAuto(ctx, documentID, tenantID, rendered)
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
// Read-only aggregate queries backing the GoBD "Verfahrensdokumentation"
|
||||
// draft generator (internal/api/compliance_handlers.go). No schema of its own —
|
||||
// nothing here writes, so there is no initSchema and no migration file.
|
||||
//
|
||||
// Every query is strictly scoped to one tenant_id (application-side
|
||||
// multi-tenancy, no Postgres RLS): the generated document is a per-tenant
|
||||
// artefact and must never mix data of two tenants.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// ComplianceStats are the aggregate key figures embedded in the generated
|
||||
// Verfahrensdokumentation draft. All counters refer to exactly one tenant.
|
||||
type ComplianceStats struct {
|
||||
Documents int64 // active documents (deleted_at IS NULL)
|
||||
DocumentsInTrash int64 // soft-deleted, not yet finally deleted
|
||||
DocumentsWithRetain int64 // active documents carrying a retain_until date
|
||||
PermissionGroups int64
|
||||
DocTypeGrants int64
|
||||
TagGrants int64
|
||||
DocumentGrants int64
|
||||
DeleteRequestsByStat map[string]int64 // status -> count
|
||||
}
|
||||
|
||||
// ComplianceStatsForTenant collects the aggregate figures for one tenant.
|
||||
func (s *Store) ComplianceStatsForTenant(ctx context.Context, tenantID int64) (*ComplianceStats, error) {
|
||||
st := &ComplianceStats{DeleteRequestsByStat: map[string]int64{}}
|
||||
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE deleted_at IS NULL),
|
||||
COUNT(*) FILTER (WHERE deleted_at IS NOT NULL),
|
||||
COUNT(*) FILTER (WHERE deleted_at IS NULL AND retain_until IS NOT NULL)
|
||||
FROM documents WHERE tenant_id = $1
|
||||
`, tenantID).Scan(&st.Documents, &st.DocumentsInTrash, &st.DocumentsWithRetain)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: compliance document stats: %w", err)
|
||||
}
|
||||
|
||||
err = s.db.QueryRow(ctx, `
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM permission_groups WHERE tenant_id = $1),
|
||||
(SELECT COUNT(*) FROM document_type_grants WHERE tenant_id = $1),
|
||||
(SELECT COUNT(*) FROM tag_grants WHERE tenant_id = $1),
|
||||
(SELECT COUNT(*) FROM document_grants WHERE tenant_id = $1)
|
||||
`, tenantID).Scan(&st.PermissionGroups, &st.DocTypeGrants, &st.TagGrants, &st.DocumentGrants)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: compliance grant stats: %w", err)
|
||||
}
|
||||
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT status, COUNT(*) FROM document_delete_requests
|
||||
WHERE tenant_id = $1 GROUP BY status ORDER BY status
|
||||
`, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: compliance delete-request stats: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var status string
|
||||
var n int64
|
||||
if err := rows.Scan(&status, &n); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan compliance delete-request stats: %w", err)
|
||||
}
|
||||
st.DeleteRequestsByStat[status] = n
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("storage: compliance delete-request stats: %w", err)
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
@@ -0,0 +1,646 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
// ErrCustomFieldNotFound is returned when a custom-field definition lookup,
|
||||
// update or delete does not match any row owned by the caller's tenant.
|
||||
var ErrCustomFieldNotFound = errors.New("storage: custom field not found or not owned by tenant")
|
||||
|
||||
// ErrDuplicateCustomFieldName is returned when a tenant already has a custom
|
||||
// field with the same name (UNIQUE(tenant_id, name)).
|
||||
var ErrDuplicateCustomFieldName = errors.New("storage: custom field with this name already exists for tenant")
|
||||
|
||||
// ErrCustomFieldInUse is returned by DeleteCustomFieldDef when values still
|
||||
// reference the field — the caller translates this into an HTTP 409.
|
||||
var ErrCustomFieldInUse = errors.New("storage: custom field still has values and cannot be deleted")
|
||||
|
||||
// ErrRequiredFieldMissing is returned by SetDocumentFieldValues when a field
|
||||
// marked required for the document's document_type has no value supplied.
|
||||
var ErrRequiredFieldMissing = errors.New("storage: required custom field missing a value")
|
||||
|
||||
// validFieldTypes mirrors the CHECK constraint on custom_field_defs.field_type.
|
||||
var validFieldTypes = map[string]bool{
|
||||
"text": true, "number": true, "date": true,
|
||||
"boolean": true, "enum": true, "monetary": true,
|
||||
}
|
||||
|
||||
// CustomFieldDef is a tenant-scoped custom-field definition.
|
||||
type CustomFieldDef struct {
|
||||
ID int64 `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
FieldType string `json:"field_type"`
|
||||
EnumOptions []string `json:"enum_options,omitempty"`
|
||||
Currency string `json:"currency,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// CustomFieldDefRequest holds create parameters for a custom-field definition.
|
||||
type CustomFieldDefRequest struct {
|
||||
Name string
|
||||
Label string
|
||||
FieldType string
|
||||
EnumOptions []string
|
||||
Currency string
|
||||
}
|
||||
|
||||
// DocumentTypeField is a custom field assigned to a document type, carrying
|
||||
// the assignment metadata (required/visible/sort_order) plus the resolved
|
||||
// field definition.
|
||||
type DocumentTypeField struct {
|
||||
FieldID int64 `json:"field_id"`
|
||||
Required bool `json:"required"`
|
||||
Visible bool `json:"visible"`
|
||||
SortOrder int `json:"sort_order"`
|
||||
Field CustomFieldDef `json:"field"`
|
||||
}
|
||||
|
||||
// DocumentTypeFieldAssignment is one entry of a bulk PUT replacing a document
|
||||
// type's field assignments.
|
||||
type DocumentTypeFieldAssignment struct {
|
||||
FieldID int64
|
||||
Required bool
|
||||
Visible bool
|
||||
SortOrder int
|
||||
}
|
||||
|
||||
// DocumentFieldValue is a single custom-field value on a document, with the
|
||||
// value carried in the type-appropriate column.
|
||||
type DocumentFieldValue struct {
|
||||
FieldID int64 `json:"field_id"`
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
FieldType string `json:"field_type"`
|
||||
Currency string `json:"currency,omitempty"`
|
||||
ValueText *string `json:"value_text,omitempty"`
|
||||
ValueNumber *float64 `json:"value_number,omitempty"`
|
||||
ValueDate *time.Time `json:"value_date,omitempty"`
|
||||
ValueBool *bool `json:"value_bool,omitempty"`
|
||||
}
|
||||
|
||||
// DocumentFieldValueInput is one supplied value in a batch PUT. Exactly one of
|
||||
// the value pointers is expected to be populated (matching the field's type).
|
||||
type DocumentFieldValueInput struct {
|
||||
FieldID int64 `json:"field_id"`
|
||||
ValueText *string `json:"value_text,omitempty"`
|
||||
ValueNumber *float64 `json:"value_number,omitempty"`
|
||||
ValueDate *string `json:"value_date,omitempty"` // ISO date "2006-01-02"
|
||||
ValueBool *bool `json:"value_bool,omitempty"`
|
||||
}
|
||||
|
||||
// initCustomFieldsSchema creates the custom_field_defs / document_type_fields /
|
||||
// document_field_values tables. Idempotent, called from (*Store).initSchema.
|
||||
// Documented (not executed) in migrations/006_custom_fields.sql.
|
||||
func (s *Store) initCustomFieldsSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS custom_field_defs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
label TEXT NOT NULL,
|
||||
field_type TEXT NOT NULL CHECK (field_type IN ('text','number','date','boolean','enum','monetary')),
|
||||
enum_options JSONB,
|
||||
currency TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_type_fields (
|
||||
doc_type_id BIGINT NOT NULL REFERENCES document_types(id) ON DELETE CASCADE,
|
||||
field_id BIGINT NOT NULL REFERENCES custom_field_defs(id) ON DELETE CASCADE,
|
||||
required BOOLEAN NOT NULL DEFAULT false,
|
||||
visible BOOLEAN NOT NULL DEFAULT true,
|
||||
sort_order INT NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (doc_type_id, field_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_field_values (
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
field_id BIGINT NOT NULL REFERENCES custom_field_defs(id) ON DELETE CASCADE,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
value_text TEXT,
|
||||
value_number NUMERIC,
|
||||
value_date DATE,
|
||||
value_bool BOOLEAN,
|
||||
PRIMARY KEY (document_id, field_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_dfv_tenant_field ON document_field_values(tenant_id, field_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_dfv_field_text ON document_field_values(field_id, value_text);
|
||||
CREATE INDEX IF NOT EXISTS idx_dfv_field_number ON document_field_values(field_id, value_number);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create custom fields tables: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func scanCustomFieldDef(row interface {
|
||||
Scan(dest ...any) error
|
||||
}) (*CustomFieldDef, error) {
|
||||
var d CustomFieldDef
|
||||
var enumRaw []byte
|
||||
var currency *string
|
||||
if err := row.Scan(&d.ID, &d.TenantID, &d.Name, &d.Label, &d.FieldType, &enumRaw, ¤cy, &d.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(enumRaw) > 0 {
|
||||
if err := json.Unmarshal(enumRaw, &d.EnumOptions); err != nil {
|
||||
return nil, fmt.Errorf("storage: unmarshal enum_options: %w", err)
|
||||
}
|
||||
}
|
||||
if currency != nil {
|
||||
d.Currency = *currency
|
||||
}
|
||||
return &d, nil
|
||||
}
|
||||
|
||||
// ListCustomFieldDefs returns all custom-field definitions for a tenant.
|
||||
func (s *Store) ListCustomFieldDefs(ctx context.Context, tenantID int64) ([]CustomFieldDef, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, tenant_id, name, label, field_type, enum_options, currency, created_at
|
||||
FROM custom_field_defs WHERE tenant_id = $1 ORDER BY name ASC
|
||||
`, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list custom fields: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]CustomFieldDef, 0)
|
||||
for rows.Next() {
|
||||
d, err := scanCustomFieldDef(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: scan custom field: %w", err)
|
||||
}
|
||||
out = append(out, *d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// GetCustomFieldDef returns one custom-field definition, scoped to tenant.
|
||||
func (s *Store) GetCustomFieldDef(ctx context.Context, id, tenantID int64) (*CustomFieldDef, error) {
|
||||
row := s.db.QueryRow(ctx, `
|
||||
SELECT id, tenant_id, name, label, field_type, enum_options, currency, created_at
|
||||
FROM custom_field_defs WHERE id = $1 AND tenant_id = $2
|
||||
`, id, tenantID)
|
||||
d, err := scanCustomFieldDef(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrCustomFieldNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("storage: get custom field: %w", err)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// CreateCustomFieldDef inserts a new custom-field definition.
|
||||
func (s *Store) CreateCustomFieldDef(ctx context.Context, tenantID int64, req CustomFieldDefRequest) (*CustomFieldDef, error) {
|
||||
if !validFieldTypes[req.FieldType] {
|
||||
return nil, fmt.Errorf("storage: invalid field_type %q", req.FieldType)
|
||||
}
|
||||
var enumRaw []byte
|
||||
if len(req.EnumOptions) > 0 {
|
||||
b, err := json.Marshal(req.EnumOptions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: marshal enum_options: %w", err)
|
||||
}
|
||||
enumRaw = b
|
||||
}
|
||||
row := s.db.QueryRow(ctx, `
|
||||
INSERT INTO custom_field_defs (tenant_id, name, label, field_type, enum_options, currency)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
RETURNING id, tenant_id, name, label, field_type, enum_options, currency, created_at
|
||||
`, tenantID, req.Name, req.Label, req.FieldType, enumRaw, nullIfEmpty(req.Currency))
|
||||
d, err := scanCustomFieldDef(row)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return nil, ErrDuplicateCustomFieldName
|
||||
}
|
||||
return nil, fmt.Errorf("storage: create custom field: %w", err)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// UpdateCustomFieldDef updates the label, enum_options and currency of a
|
||||
// custom-field definition. Name and field_type are immutable (they anchor
|
||||
// stored values), matching the API contract. Scoped to tenant ownership.
|
||||
func (s *Store) UpdateCustomFieldDef(ctx context.Context, id, tenantID int64, label string, enumOptions []string, currency string) (*CustomFieldDef, error) {
|
||||
var enumRaw []byte
|
||||
if len(enumOptions) > 0 {
|
||||
b, err := json.Marshal(enumOptions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: marshal enum_options: %w", err)
|
||||
}
|
||||
enumRaw = b
|
||||
}
|
||||
row := s.db.QueryRow(ctx, `
|
||||
UPDATE custom_field_defs SET label = $1, enum_options = $2, currency = $3
|
||||
WHERE id = $4 AND tenant_id = $5
|
||||
RETURNING id, tenant_id, name, label, field_type, enum_options, currency, created_at
|
||||
`, label, enumRaw, nullIfEmpty(currency), id, tenantID)
|
||||
d, err := scanCustomFieldDef(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrCustomFieldNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("storage: update custom field: %w", err)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// DeleteCustomFieldDef deletes a custom-field definition, but only if no
|
||||
// document_field_values reference it. Returns ErrCustomFieldInUse otherwise.
|
||||
// Scoped to tenant ownership.
|
||||
func (s *Store) DeleteCustomFieldDef(ctx context.Context, id, tenantID int64) error {
|
||||
// Ownership check first — distinguishes 404 from 409.
|
||||
if _, err := s.GetCustomFieldDef(ctx, id, tenantID); err != nil {
|
||||
return err
|
||||
}
|
||||
var inUse bool
|
||||
if err := s.db.QueryRow(ctx, `
|
||||
SELECT EXISTS(SELECT 1 FROM document_field_values WHERE field_id = $1 AND tenant_id = $2)
|
||||
`, id, tenantID).Scan(&inUse); err != nil {
|
||||
return fmt.Errorf("storage: check custom field usage: %w", err)
|
||||
}
|
||||
if inUse {
|
||||
return ErrCustomFieldInUse
|
||||
}
|
||||
tag, err := s.db.Exec(ctx, `DELETE FROM custom_field_defs WHERE id = $1 AND tenant_id = $2`, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: delete custom field: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrCustomFieldNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ownsDocumentType returns true if the document type is owned by the tenant.
|
||||
func (s *Store) ownsDocumentType(ctx context.Context, docTypeID, tenantID int64) (bool, error) {
|
||||
var ok bool
|
||||
err := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM document_types WHERE id = $1 AND tenant_id = $2)`, docTypeID, tenantID).Scan(&ok)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("storage: check document type ownership: %w", err)
|
||||
}
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
// ListDocumentTypeFields returns the custom fields assigned to a document type
|
||||
// (with required/visible/sort_order), joined to their definitions. Scoped to
|
||||
// tenant ownership of the document type.
|
||||
func (s *Store) ListDocumentTypeFields(ctx context.Context, docTypeID, tenantID int64) ([]DocumentTypeField, error) {
|
||||
owns, err := s.ownsDocumentType(ctx, docTypeID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !owns {
|
||||
return nil, ErrTaxonomyNotFound
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT dtf.field_id, dtf.required, dtf.visible, dtf.sort_order,
|
||||
f.id, f.tenant_id, f.name, f.label, f.field_type, f.enum_options, f.currency, f.created_at
|
||||
FROM document_type_fields dtf
|
||||
JOIN custom_field_defs f ON f.id = dtf.field_id
|
||||
WHERE dtf.doc_type_id = $1 AND f.tenant_id = $2
|
||||
ORDER BY dtf.sort_order ASC, f.name ASC
|
||||
`, docTypeID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list document type fields: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]DocumentTypeField, 0)
|
||||
for rows.Next() {
|
||||
var a DocumentTypeField
|
||||
var f CustomFieldDef
|
||||
var enumRaw []byte
|
||||
var currency *string
|
||||
if err := rows.Scan(&a.FieldID, &a.Required, &a.Visible, &a.SortOrder,
|
||||
&f.ID, &f.TenantID, &f.Name, &f.Label, &f.FieldType, &enumRaw, ¤cy, &f.CreatedAt); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan document type field: %w", err)
|
||||
}
|
||||
if len(enumRaw) > 0 {
|
||||
if err := json.Unmarshal(enumRaw, &f.EnumOptions); err != nil {
|
||||
return nil, fmt.Errorf("storage: unmarshal enum_options: %w", err)
|
||||
}
|
||||
}
|
||||
if currency != nil {
|
||||
f.Currency = *currency
|
||||
}
|
||||
a.Field = f
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SetDocumentTypeFields replaces the complete set of field assignments for a
|
||||
// document type (bulk PUT). All referenced fields must belong to the tenant.
|
||||
// Scoped to tenant ownership of the document type.
|
||||
func (s *Store) SetDocumentTypeFields(ctx context.Context, docTypeID, tenantID int64, assignments []DocumentTypeFieldAssignment) error {
|
||||
owns, err := s.ownsDocumentType(ctx, docTypeID, tenantID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !owns {
|
||||
return ErrTaxonomyNotFound
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: begin set document type fields: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM document_type_fields WHERE doc_type_id = $1`, docTypeID); err != nil {
|
||||
return fmt.Errorf("storage: clear document type fields: %w", err)
|
||||
}
|
||||
for _, a := range assignments {
|
||||
// Verify field ownership by tenant before linking.
|
||||
var ok bool
|
||||
if err := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM custom_field_defs WHERE id = $1 AND tenant_id = $2)`, a.FieldID, tenantID).Scan(&ok); err != nil {
|
||||
return fmt.Errorf("storage: check field ownership: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: field_id %d", ErrCustomFieldNotFound, a.FieldID)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO document_type_fields (doc_type_id, field_id, required, visible, sort_order)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (doc_type_id, field_id) DO UPDATE
|
||||
SET required = EXCLUDED.required, visible = EXCLUDED.visible, sort_order = EXCLUDED.sort_order
|
||||
`, docTypeID, a.FieldID, a.Required, a.Visible, a.SortOrder); err != nil {
|
||||
return fmt.Errorf("storage: insert document type field: %w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("storage: commit set document type fields: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListDocumentFieldValues returns the custom-field values stored on a document
|
||||
// (joined to their definitions), scoped to tenant. Ownership of the document
|
||||
// must be verified by the caller.
|
||||
func (s *Store) ListDocumentFieldValues(ctx context.Context, documentID, tenantID int64) ([]DocumentFieldValue, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT v.field_id, f.name, f.label, f.field_type, f.currency,
|
||||
v.value_text, v.value_number, v.value_date, v.value_bool
|
||||
FROM document_field_values v
|
||||
JOIN custom_field_defs f ON f.id = v.field_id
|
||||
WHERE v.document_id = $1 AND v.tenant_id = $2
|
||||
ORDER BY f.name ASC
|
||||
`, documentID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list document field values: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]DocumentFieldValue, 0)
|
||||
for rows.Next() {
|
||||
var v DocumentFieldValue
|
||||
var currency *string
|
||||
if err := rows.Scan(&v.FieldID, &v.Name, &v.Label, &v.FieldType, ¤cy,
|
||||
&v.ValueText, &v.ValueNumber, &v.ValueDate, &v.ValueBool); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan document field value: %w", err)
|
||||
}
|
||||
if currency != nil {
|
||||
v.Currency = *currency
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SetDocumentFieldValues sets (upserts) a batch of custom-field values on a
|
||||
// document and deletes any values not present in the batch. It validates each
|
||||
// field against its type and enforces required fields for the document's
|
||||
// document_type server-side. Returns the names of fields whose value changed
|
||||
// (for audit logging). Scoped to tenant. Ownership of the document must be
|
||||
// verified by the caller.
|
||||
func (s *Store) SetDocumentFieldValues(ctx context.Context, documentID, tenantID int64, inputs []DocumentFieldValueInput) ([]string, error) {
|
||||
// Load the tenant's field definitions for type resolution.
|
||||
defs, err := s.ListCustomFieldDefs(ctx, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defByID := make(map[int64]CustomFieldDef, len(defs))
|
||||
for _, d := range defs {
|
||||
defByID[d.ID] = d
|
||||
}
|
||||
|
||||
// Resolve the document's document_type_id to know which fields are required.
|
||||
var docTypeID *int64
|
||||
if err := s.db.QueryRow(ctx, `SELECT doc_type_id FROM documents WHERE id = $1 AND tenant_id = $2`, documentID, tenantID).Scan(&docTypeID); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, fmt.Errorf("storage: document not found or not owned by tenant")
|
||||
}
|
||||
return nil, fmt.Errorf("storage: resolve document doc_type: %w", err)
|
||||
}
|
||||
|
||||
// Build the set of supplied non-empty field values keyed by field_id.
|
||||
type resolved struct {
|
||||
text *string
|
||||
number *float64
|
||||
date *time.Time
|
||||
bl *bool
|
||||
}
|
||||
supplied := make(map[int64]resolved, len(inputs))
|
||||
for _, in := range inputs {
|
||||
def, ok := defByID[in.FieldID]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: field_id %d", ErrCustomFieldNotFound, in.FieldID)
|
||||
}
|
||||
var r resolved
|
||||
switch def.FieldType {
|
||||
case "text":
|
||||
r.text = in.ValueText
|
||||
case "enum":
|
||||
if in.ValueText != nil && *in.ValueText != "" {
|
||||
if len(def.EnumOptions) > 0 && !containsString(def.EnumOptions, *in.ValueText) {
|
||||
return nil, fmt.Errorf("storage: value %q not in enum options for field %q", *in.ValueText, def.Name)
|
||||
}
|
||||
}
|
||||
r.text = in.ValueText
|
||||
case "number", "monetary":
|
||||
r.number = in.ValueNumber
|
||||
case "date":
|
||||
if in.ValueDate != nil && *in.ValueDate != "" {
|
||||
t, err := time.Parse("2006-01-02", *in.ValueDate)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: invalid date %q for field %q: %w", *in.ValueDate, def.Name, err)
|
||||
}
|
||||
r.date = &t
|
||||
}
|
||||
case "boolean":
|
||||
r.bl = in.ValueBool
|
||||
}
|
||||
supplied[in.FieldID] = r
|
||||
}
|
||||
|
||||
// Required-field validation against the document's type assignments.
|
||||
if docTypeID != nil {
|
||||
reqRows, err := s.db.Query(ctx, `
|
||||
SELECT dtf.field_id FROM document_type_fields dtf
|
||||
JOIN custom_field_defs f ON f.id = dtf.field_id
|
||||
WHERE dtf.doc_type_id = $1 AND f.tenant_id = $2 AND dtf.required = true
|
||||
`, *docTypeID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: load required fields: %w", err)
|
||||
}
|
||||
var requiredIDs []int64
|
||||
for reqRows.Next() {
|
||||
var fid int64
|
||||
if err := reqRows.Scan(&fid); err != nil {
|
||||
reqRows.Close()
|
||||
return nil, fmt.Errorf("storage: scan required field: %w", err)
|
||||
}
|
||||
requiredIDs = append(requiredIDs, fid)
|
||||
}
|
||||
reqRows.Close()
|
||||
if err := reqRows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, fid := range requiredIDs {
|
||||
r, ok := supplied[fid]
|
||||
if !ok || isEmptyResolved(r.text, r.number, r.date, r.bl) {
|
||||
def := defByID[fid]
|
||||
return nil, fmt.Errorf("%w: %s", ErrRequiredFieldMissing, def.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Determine current values to compute the changed-field set for audit.
|
||||
existing, err := s.ListDocumentFieldValues(ctx, documentID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
existingByID := make(map[int64]DocumentFieldValue, len(existing))
|
||||
for _, e := range existing {
|
||||
existingByID[e.FieldID] = e
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: begin set document field values: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var changed []string
|
||||
keep := make(map[int64]bool, len(supplied))
|
||||
for fid, r := range supplied {
|
||||
def := defByID[fid]
|
||||
// Empty value => treat as deletion (handled by the not-kept sweep).
|
||||
if isEmptyResolved(r.text, r.number, r.date, r.bl) {
|
||||
continue
|
||||
}
|
||||
keep[fid] = true
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO document_field_values (document_id, field_id, tenant_id, value_text, value_number, value_date, value_bool)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (document_id, field_id) DO UPDATE
|
||||
SET value_text = EXCLUDED.value_text, value_number = EXCLUDED.value_number,
|
||||
value_date = EXCLUDED.value_date, value_bool = EXCLUDED.value_bool
|
||||
`, documentID, fid, tenantID, r.text, r.number, r.date, r.bl); err != nil {
|
||||
return nil, fmt.Errorf("storage: upsert document field value: %w", err)
|
||||
}
|
||||
if changedValue(existingByID[fid], r.text, r.number, r.date, r.bl) {
|
||||
changed = append(changed, def.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete values that were present but are no longer supplied (or were
|
||||
// supplied empty). Only within this tenant/document.
|
||||
for fid, e := range existingByID {
|
||||
if keep[fid] {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM document_field_values WHERE document_id = $1 AND field_id = $2 AND tenant_id = $3`, documentID, fid, tenantID); err != nil {
|
||||
return nil, fmt.Errorf("storage: delete document field value: %w", err)
|
||||
}
|
||||
changed = append(changed, e.Name)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, fmt.Errorf("storage: commit set document field values: %w", err)
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func containsString(list []string, s string) bool {
|
||||
for _, v := range list {
|
||||
if v == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isEmptyResolved(text *string, number *float64, date *time.Time, bl *bool) bool {
|
||||
if text != nil && *text != "" {
|
||||
return false
|
||||
}
|
||||
if number != nil {
|
||||
return false
|
||||
}
|
||||
if date != nil {
|
||||
return false
|
||||
}
|
||||
if bl != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func changedValue(prev DocumentFieldValue, text *string, number *float64, date *time.Time, bl *bool) bool {
|
||||
if !ptrEqStr(prev.ValueText, text) {
|
||||
return true
|
||||
}
|
||||
if !ptrEqFloat(prev.ValueNumber, number) {
|
||||
return true
|
||||
}
|
||||
if !ptrEqDate(prev.ValueDate, date) {
|
||||
return true
|
||||
}
|
||||
if !ptrEqBool(prev.ValueBool, bl) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ptrEqStr(a, b *string) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == nil && b == nil
|
||||
}
|
||||
return *a == *b
|
||||
}
|
||||
|
||||
func ptrEqFloat(a, b *float64) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == nil && b == nil
|
||||
}
|
||||
return *a == *b
|
||||
}
|
||||
|
||||
func ptrEqBool(a, b *bool) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == nil && b == nil
|
||||
}
|
||||
return *a == *b
|
||||
}
|
||||
|
||||
func ptrEqDate(a, b *time.Time) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == nil && b == nil
|
||||
}
|
||||
return a.Year() == b.Year() && a.Month() == b.Month() && a.Day() == b.Day()
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
// Dashboard aggregation store. MVP: a single struct of live COUNT/GROUP BY
|
||||
// queries per tenant, no caching layer and no materialized views. All queries
|
||||
// are tenant-scoped (WHERE tenant_id = $1) exactly like the rest of the store.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// DashboardStats is the aggregated key-figure snapshot for one tenant, served
|
||||
// by GET /api/dashboard.
|
||||
type DashboardStats struct {
|
||||
TotalDocuments int64 `json:"total_documents"`
|
||||
DocumentsThisMonth int64 `json:"documents_this_month"`
|
||||
TrashCount int64 `json:"trash_count"`
|
||||
PendingDeleteRequests int64 `json:"pending_delete_requests"`
|
||||
RemindersDue int64 `json:"reminders_due"`
|
||||
RemindersUpcoming7d int64 `json:"reminders_upcoming_7d"`
|
||||
RetentionExpiring30d int64 `json:"retention_expiring_30d"`
|
||||
DocumentsByType []DocumentTypeCount `json:"documents_by_type"`
|
||||
}
|
||||
|
||||
// DocumentTypeCount is one entry of the documents_by_type breakdown.
|
||||
type DocumentTypeCount struct {
|
||||
DocumentTypeName string `json:"document_type_name"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
||||
// GetDashboardStats computes the aggregated dashboard key figures for a tenant.
|
||||
// Reminders are additionally scoped to the requesting user (reminders are
|
||||
// per-user like in ListReminders); the document/trash/retention figures are
|
||||
// tenant-wide.
|
||||
func (s *Store) GetDashboardStats(ctx context.Context, tenantID, userID int64) (*DashboardStats, error) {
|
||||
var stats DashboardStats
|
||||
|
||||
// Documents: active count + this-calendar-month count in one scan.
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE deleted_at IS NULL),
|
||||
COUNT(*) FILTER (WHERE deleted_at IS NULL AND created_at >= date_trunc('month', now())),
|
||||
COUNT(*) FILTER (WHERE deleted_at IS NOT NULL)
|
||||
FROM documents WHERE tenant_id = $1
|
||||
`, tenantID).Scan(&stats.TotalDocuments, &stats.DocumentsThisMonth, &stats.TrashCount)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: dashboard document counts: %w", err)
|
||||
}
|
||||
|
||||
// Open delete requests (awaiting confirmation or blocked by retention).
|
||||
err = s.db.QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM document_delete_requests
|
||||
WHERE tenant_id = $1 AND status IN ('pending', 'blocked_retention')
|
||||
`, tenantID).Scan(&stats.PendingDeleteRequests)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: dashboard pending delete requests: %w", err)
|
||||
}
|
||||
|
||||
// Reminders (per user): due/overdue and upcoming within 7 days. "Not done"
|
||||
// means still status='open' (see reminders.go status semantics).
|
||||
err = s.db.QueryRow(ctx, `
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE due_date <= now()),
|
||||
COUNT(*) FILTER (WHERE due_date > now() AND due_date <= now() + interval '7 days')
|
||||
FROM reminders WHERE tenant_id = $1 AND user_id = $2 AND status = 'open'
|
||||
`, tenantID, userID).Scan(&stats.RemindersDue, &stats.RemindersUpcoming7d)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: dashboard reminders: %w", err)
|
||||
}
|
||||
|
||||
// Retention expiring within the next 30 days (active documents only).
|
||||
err = s.db.QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM documents
|
||||
WHERE tenant_id = $1 AND deleted_at IS NULL
|
||||
AND retain_until IS NOT NULL
|
||||
AND retain_until >= current_date
|
||||
AND retain_until <= current_date + 30
|
||||
`, tenantID).Scan(&stats.RetentionExpiring30d)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: dashboard retention expiring: %w", err)
|
||||
}
|
||||
|
||||
// Documents by type — top 5 by count. Uses the structured document_types
|
||||
// entity via doc_type_id, falling back to the deprecated free-text doc_type
|
||||
// for Bestandsschutz, and "(ohne Typ)" when neither is set.
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT COALESCE(dt.name, NULLIF(d.doc_type, ''), '(ohne Typ)') AS type_name, COUNT(*) AS cnt
|
||||
FROM documents d
|
||||
LEFT JOIN document_types dt ON dt.id = d.doc_type_id
|
||||
WHERE d.tenant_id = $1 AND d.deleted_at IS NULL
|
||||
GROUP BY type_name
|
||||
ORDER BY cnt DESC, type_name ASC
|
||||
LIMIT 5
|
||||
`, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: dashboard documents by type: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var c DocumentTypeCount
|
||||
if err := rows.Scan(&c.DocumentTypeName, &c.Count); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan documents by type: %w", err)
|
||||
}
|
||||
stats.DocumentsByType = append(stats.DocumentsByType, c)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("storage: dashboard documents by type rows: %w", err)
|
||||
}
|
||||
if stats.DocumentsByType == nil {
|
||||
stats.DocumentsByType = []DocumentTypeCount{}
|
||||
}
|
||||
|
||||
return &stats, nil
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// documentDateMinYear mirrors dateMinYear in internal/api/date_extraction.go.
|
||||
// The whole scoring logic below is duplicated (rather than shared) because the
|
||||
// storage package must not import internal/api (that would create an import
|
||||
// cycle: api already depends on storage). Kept in sync with the api heuristic —
|
||||
// the same duplication pattern as heuristicTitle vs. titleFromOCRText.
|
||||
const (
|
||||
documentDateMinYear = 1990
|
||||
documentDateFutureToleranceDays = 2
|
||||
documentDateWindowRadius = 40
|
||||
documentDateScoreNoContext = 0.4
|
||||
)
|
||||
|
||||
// documentDateKeyword mirrors dateKeyword in internal/api/date_extraction.go.
|
||||
type documentDateKeyword struct {
|
||||
word string
|
||||
score float64
|
||||
}
|
||||
|
||||
var documentDateKeywords = []documentDateKeyword{
|
||||
{"rechnungsdatum", 0.9},
|
||||
{"belegdatum", 0.9},
|
||||
{"ausstellungsdatum", 0.9},
|
||||
{"rechnung vom", 0.9},
|
||||
{"beleg vom", 0.9},
|
||||
{"datum", 0.75},
|
||||
{"vom", 0.55},
|
||||
}
|
||||
|
||||
// documentDateGermanMonths mirrors dateGermanMonths in the api package.
|
||||
var documentDateGermanMonths = map[string]int{
|
||||
"januar": 1, "jan": 1,
|
||||
"februar": 2, "feb": 2,
|
||||
"märz": 3, "maerz": 3, "mär": 3, "mrz": 3,
|
||||
"april": 4, "apr": 4,
|
||||
"mai": 5,
|
||||
"juni": 6, "jun": 6,
|
||||
"juli": 7, "jul": 7,
|
||||
"august": 8, "aug": 8,
|
||||
"september": 9, "sep": 9, "sept": 9,
|
||||
"oktober": 10, "okt": 10,
|
||||
"november": 11, "nov": 11,
|
||||
"dezember": 12, "dez": 12,
|
||||
}
|
||||
|
||||
// documentDateCandidateRe mirrors dateCandidateRe in
|
||||
// internal/api/date_extraction.go. Recognised: DD.MM.YYYY, DD.MM.YY,
|
||||
// DD/MM/YYYY, YYYY-MM-DD and spelled-out German month names ("15. März 2026").
|
||||
var documentDateCandidateRe = regexp.MustCompile(
|
||||
`(?i)(?:\b(?P<gd>\d{1,2})\.(?P<gm>\d{1,2})\.(?P<gy>\d{4}|\d{2})\b)` +
|
||||
`|(?:\b(?P<sd>\d{1,2})/(?P<sm>\d{1,2})/(?P<sy>\d{4}|\d{2})\b)` +
|
||||
`|(?:\b(?P<iy>\d{4})-(?P<im>\d{1,2})-(?P<id>\d{1,2})\b)` +
|
||||
`|(?:\b(?P<td>\d{1,2})\.?\s+(?P<tmon>[A-Za-zäöüÄÖÜ]+)\.?\s+(?P<ty>\d{4})\b)`,
|
||||
)
|
||||
|
||||
// documentDateScoreForPosition mirrors scoreForDatePosition in the api package.
|
||||
func documentDateScoreForPosition(lowerText string, start int) float64 {
|
||||
lo := start - documentDateWindowRadius
|
||||
if lo < 0 {
|
||||
lo = 0
|
||||
}
|
||||
hi := start + documentDateWindowRadius
|
||||
if hi > len(lowerText) {
|
||||
hi = len(lowerText)
|
||||
}
|
||||
window := lowerText[lo:hi]
|
||||
for _, kw := range documentDateKeywords {
|
||||
if strings.Contains(window, kw.word) {
|
||||
return kw.score
|
||||
}
|
||||
}
|
||||
return documentDateScoreNoContext
|
||||
}
|
||||
|
||||
// documentDateParseMatch mirrors parseDateMatch in the api package.
|
||||
func documentDateParseMatch(names, m []string) (time.Time, bool) {
|
||||
now := time.Now()
|
||||
maxDate := now.AddDate(0, 0, documentDateFutureToleranceDays)
|
||||
var day, month, year int
|
||||
var monthName string
|
||||
for i, name := range names {
|
||||
if m[i] == "" {
|
||||
continue
|
||||
}
|
||||
switch name {
|
||||
case "gd", "sd", "id", "td":
|
||||
day, _ = strconv.Atoi(m[i])
|
||||
case "gm", "sm", "im":
|
||||
month, _ = strconv.Atoi(m[i])
|
||||
case "gy", "sy":
|
||||
y, _ := strconv.Atoi(m[i])
|
||||
if len(m[i]) == 2 {
|
||||
y += 2000
|
||||
}
|
||||
year = y
|
||||
case "iy", "ty":
|
||||
year, _ = strconv.Atoi(m[i])
|
||||
case "tmon":
|
||||
monthName = m[i]
|
||||
}
|
||||
}
|
||||
if monthName != "" {
|
||||
mn, ok := documentDateGermanMonths[strings.ToLower(monthName)]
|
||||
if !ok {
|
||||
return time.Time{}, false
|
||||
}
|
||||
month = mn
|
||||
}
|
||||
if year < documentDateMinYear {
|
||||
return time.Time{}, false
|
||||
}
|
||||
if month < 1 || month > 12 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
if day < 1 || day > 31 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
d := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC)
|
||||
if d.Day() != day || int(d.Month()) != month || d.Year() != year {
|
||||
return time.Time{}, false
|
||||
}
|
||||
if d.After(maxDate) {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
|
||||
// documentDateFromTextWithScore mirrors extractDocumentDateWithScore in the api
|
||||
// package. Returns the best belegdatum candidate and its confidence.
|
||||
func documentDateFromTextWithScore(ocrText string) (best time.Time, score float64, found bool) {
|
||||
if ocrText == "" {
|
||||
return time.Time{}, 0, false
|
||||
}
|
||||
lower := strings.ToLower(ocrText)
|
||||
idxMatches := documentDateCandidateRe.FindAllStringSubmatchIndex(ocrText, -1)
|
||||
names := documentDateCandidateRe.SubexpNames()
|
||||
for _, loc := range idxMatches {
|
||||
m := make([]string, len(names))
|
||||
for g := range names {
|
||||
s, e := loc[2*g], loc[2*g+1]
|
||||
if s >= 0 {
|
||||
m[g] = ocrText[s:e]
|
||||
}
|
||||
}
|
||||
d, ok := documentDateParseMatch(names, m)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
sc := documentDateScoreForPosition(lower, loc[0])
|
||||
if !found || sc > score {
|
||||
best, score, found = d, sc, true
|
||||
}
|
||||
}
|
||||
return best, score, found
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// ErrNoteNotFound is returned when a tenant-scoped note lookup/delete affects
|
||||
// zero rows (wrong id, wrong document, or wrong tenant).
|
||||
var ErrNoteNotFound = errors.New("storage: document note not found")
|
||||
|
||||
// ErrNoteForbidden is returned by DeleteDocumentNote when the requester is
|
||||
// neither the note's author nor a domain admin.
|
||||
var ErrNoteForbidden = errors.New("storage: not allowed to delete this document note")
|
||||
|
||||
// DocumentNote is a free-text comment attached to a document (Paperless-ngx
|
||||
// inspired). Unlike custom fields (which are structured metadata), a note is
|
||||
// plain free text with an author and timestamps. Notes are not GoBD documents
|
||||
// themselves, so they are hard-deleted rather than soft-deleted — but every
|
||||
// create/delete is still recorded in the audit log for Nachvollziehbarkeit.
|
||||
type DocumentNote struct {
|
||||
ID int64 `json:"id"`
|
||||
DocumentID int64 `json:"document_id"`
|
||||
AuthorID int64 `json:"author_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// initDocumentNotesSchema creates the document_notes table. Idempotent; wired
|
||||
// into Store.initSchema after initWorkflowsSchema (see documents.go).
|
||||
func (s *Store) initDocumentNotesSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS document_notes (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id),
|
||||
tenant_id BIGINT NOT NULL,
|
||||
author_id BIGINT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_notes_document ON document_notes (document_id, tenant_id);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create document_notes table: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateDocumentNote inserts a free-text note on a document. It verifies the
|
||||
// document exists within the tenant (and is not soft-deleted) before inserting,
|
||||
// so a note can never be attached to a foreign-tenant document (IDOR guard).
|
||||
func (s *Store) CreateDocumentNote(ctx context.Context, documentID, tenantID, authorID int64, text string) (*DocumentNote, error) {
|
||||
var exists bool
|
||||
if err := s.db.QueryRow(ctx, `
|
||||
SELECT EXISTS (SELECT 1 FROM documents WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL)
|
||||
`, documentID, tenantID).Scan(&exists); err != nil {
|
||||
return nil, fmt.Errorf("storage: check document for note: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
return nil, ErrDocumentNotFound
|
||||
}
|
||||
|
||||
var n DocumentNote
|
||||
err := s.db.QueryRow(ctx, `
|
||||
INSERT INTO document_notes (document_id, tenant_id, author_id, text)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
RETURNING id, document_id, author_id, text, created_at, updated_at
|
||||
`, documentID, tenantID, authorID, text).Scan(&n.ID, &n.DocumentID, &n.AuthorID, &n.Text, &n.CreatedAt, &n.UpdatedAt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: create document note: %w", err)
|
||||
}
|
||||
return &n, nil
|
||||
}
|
||||
|
||||
// ListDocumentNotes returns all notes for a document, oldest first, scoped to
|
||||
// tenant.
|
||||
func (s *Store) ListDocumentNotes(ctx context.Context, documentID, tenantID int64) ([]DocumentNote, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, document_id, author_id, text, created_at, updated_at
|
||||
FROM document_notes
|
||||
WHERE document_id = $1 AND tenant_id = $2
|
||||
ORDER BY created_at ASC
|
||||
`, documentID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list document notes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]DocumentNote, 0)
|
||||
for rows.Next() {
|
||||
var n DocumentNote
|
||||
if err := rows.Scan(&n.ID, &n.DocumentID, &n.AuthorID, &n.Text, &n.CreatedAt, &n.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan document note: %w", err)
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// DeleteDocumentNote hard-deletes a note. Only the note's author or a domain
|
||||
// admin may delete it. Returns ErrNoteNotFound if the note does not exist for
|
||||
// the given document/tenant, or ErrNoteForbidden if the requester is not
|
||||
// permitted (checked before deletion so the caller can return 403 vs 404).
|
||||
func (s *Store) DeleteDocumentNote(ctx context.Context, id, documentID, tenantID, requesterID int64, requesterIsAdmin bool) error {
|
||||
var authorID int64
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT author_id FROM document_notes WHERE id = $1 AND document_id = $2 AND tenant_id = $3
|
||||
`, id, documentID, tenantID).Scan(&authorID)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrNoteNotFound
|
||||
}
|
||||
return fmt.Errorf("storage: lookup document note: %w", err)
|
||||
}
|
||||
if !requesterIsAdmin && authorID != requesterID {
|
||||
return ErrNoteForbidden
|
||||
}
|
||||
tag, err := s.db.Exec(ctx, `DELETE FROM document_notes WHERE id = $1 AND document_id = $2 AND tenant_id = $3`, id, documentID, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: delete document note: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrNoteNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,588 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
// ErrDuplicateContentHash is returned by CreateDocument when the tenant
|
||||
// already has an (active) document with the same content_hash — the DB-level
|
||||
// half of the duplicate-upload protection (see
|
||||
// migrations/003_documents_unique_hash.sql). The upload handler translates
|
||||
// this into an HTTP 409.
|
||||
var ErrDuplicateContentHash = errors.New("storage: document with this content hash already exists for tenant")
|
||||
|
||||
// ErrDocumentNotFound is returned when a tenant-scoped document lookup/update
|
||||
// affects zero rows (wrong id, wrong tenant, or already soft-deleted).
|
||||
var ErrDocumentNotFound = errors.New("storage: document not found")
|
||||
|
||||
// Document is the core archivdms record: a stored, indexed, and (optionally)
|
||||
// GoBD-retention-locked document. Unlike archivmail's `emails` table this is
|
||||
// intentionally generic — it is not tied to any particular ingestion source.
|
||||
//
|
||||
// Source / SourceRef are nullable placeholders for a later, purely optional
|
||||
// archivmail-pull importer (via archivmail's REST API) — no importer exists
|
||||
// yet, these columns just reserve the shape.
|
||||
type Document struct {
|
||||
ID int64 `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
Title string `json:"title"`
|
||||
DocType string `json:"doc_type,omitempty"` // deprecated free-text, kept for Bestandsschutz
|
||||
Correspondent string `json:"correspondent,omitempty"` // deprecated free-text, kept for Bestandsschutz
|
||||
DocTypeID *int64 `json:"doc_type_id,omitempty"`
|
||||
CorrespondentID *int64 `json:"correspondent_id,omitempty"`
|
||||
StoragePath string `json:"storage_path"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
OCRText string `json:"ocr_text,omitempty"`
|
||||
RetainUntil *time.Time `json:"retain_until,omitempty"`
|
||||
// DocumentDate is the recognised belegdatum (invoice/document date) parsed
|
||||
// from the OCR text at upload/reprocess time, if any. It is separate from
|
||||
// CreatedAt (the immutable scan/upload timestamp kept for GoBD
|
||||
// traceability) and drives the store/<tenant>/<yyyy>/<mm>/ archival path at
|
||||
// upload time. Nullable — nil when no plausible date was found in the text.
|
||||
DocumentDate *time.Time `json:"document_date,omitempty"`
|
||||
// DocumentDateScore is the confidence (0.4-0.9 automatic, 1.0 = manually
|
||||
// confirmed by a user via PUT .../document-date) of DocumentDate, or nil
|
||||
// when DocumentDate itself is nil / unknown (pre-existing rows from before
|
||||
// this column existed). Intended as a quality gate for the future
|
||||
// Buchhaltungs-Pull-API (only Score >= 0.75 auto-pullable) — see
|
||||
// migrations/025_document_date_score.sql. Never backfilled for existing
|
||||
// rows; NULL correctly means "no known score", not "score 0".
|
||||
DocumentDateScore *float64 `json:"document_date_score,omitempty"`
|
||||
Source string `json:"source,omitempty"` // e.g. "upload" | "archivmail_import"
|
||||
SourceRef string `json:"source_ref,omitempty"` // external ref, e.g. archivmail mail ID
|
||||
CreatedBy *int64 `json:"created_by,omitempty"`
|
||||
// TitleManuallySet is true once a user has explicitly renamed the document
|
||||
// via PATCH /api/documents/{id} (handleUpdateDocumentTitle). It gates
|
||||
// whether POST .../reprocess is allowed to overwrite the title with a
|
||||
// freshly re-derived one — see UpdateDocumentTitleAuto.
|
||||
TitleManuallySet bool `json:"title_manually_set"`
|
||||
// HasThumbnail is true once an eager preview thumbnail has been rendered
|
||||
// for this document (at upload or reprocess time, see internal/api
|
||||
// storeUploadedFile / ReprocessDocument) to
|
||||
// config.StorageConfig.ThumbnailPath()/<tenant>/<hash>.png. False for
|
||||
// documents whose thumbnail render failed/was skipped (unsupported
|
||||
// format, missing generator) or for any pre-existing document from before
|
||||
// this column existed — GET /api/documents/{id}/thumbnail still falls
|
||||
// back to on-demand lazy generation regardless of this flag, so a false
|
||||
// value never hard-blocks the preview, it's purely an optimisation hint.
|
||||
HasThumbnail bool `json:"has_thumbnail"`
|
||||
// ProcessingStatus spiegelt den Zustand der asynchronen Nachverarbeitung
|
||||
// (OCR/Taxonomie/Workflows) wider: queued | processing | done | failed
|
||||
// (siehe processing_jobs.go). Altbestand steht per Spalten-Default auf
|
||||
// 'done'. ACHTUNG: nur die Abfragen in dieser Datei (CreateDocument*,
|
||||
// GetDocument, ListDocuments) füllen das Feld — andere Selects (Suche,
|
||||
// Papierkorb, Retention) lassen es leer; "" ist vom Frontend wie 'done'
|
||||
// zu behandeln.
|
||||
ProcessingStatus string `json:"processing_status,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// CreateDocumentRequest holds parameters for creating a new document.
|
||||
type CreateDocumentRequest struct {
|
||||
TenantID int64
|
||||
Title string
|
||||
DocType string
|
||||
Correspondent string
|
||||
StoragePath string
|
||||
ContentHash string
|
||||
OCRText string
|
||||
RetainUntil *time.Time
|
||||
// DocumentDate is the recognised belegdatum from the OCR text (nil if none).
|
||||
DocumentDate *time.Time
|
||||
// DocumentDateScore is the confidence for DocumentDate, mirrors the field
|
||||
// on Document (nil if DocumentDate is nil / unknown).
|
||||
DocumentDateScore *float64
|
||||
Source string
|
||||
SourceRef string
|
||||
// CreatedBy is the uploading user's ID, if known (nil for non-interactive
|
||||
// sources like the SFTP watcher). Used so an uploader can always see
|
||||
// their own document even before any permission-group grant exists for
|
||||
// it — otherwise a plain 'user' role could upload a document and
|
||||
// immediately lose visibility of it (see RecomputeVisibility/ACL model).
|
||||
CreatedBy *int64
|
||||
}
|
||||
|
||||
func (s *Store) initSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS documents (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
doc_type TEXT,
|
||||
correspondent TEXT,
|
||||
storage_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
ocr_text TEXT,
|
||||
retain_until DATE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_documents_tenant ON documents(tenant_id);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create documents table: %w", err)
|
||||
}
|
||||
|
||||
// Duplicate protection at the DB level, on top of the filesystem-level
|
||||
// collision check performed by the upload handler (see
|
||||
// migrations/003_documents_unique_hash.sql).
|
||||
_, err = s.db.Exec(ctx, `
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_documents_tenant_hash ON documents(tenant_id, content_hash);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create documents unique hash index: %w", err)
|
||||
}
|
||||
|
||||
// Space for a later, optional archivmail-pull importer (see Document doc
|
||||
// comment). Added as nullable columns now, idempotently, no importer yet.
|
||||
_, err = s.db.Exec(ctx, `
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS source TEXT;
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS source_ref TEXT;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: alter documents table: %w", err)
|
||||
}
|
||||
|
||||
// created_by: nullable, no FK (users can be deleted independently of
|
||||
// their historical uploads, same Bestandsschutz reasoning as elsewhere
|
||||
// in this schema). Lets ListDocuments always show the uploader their own
|
||||
// document even with zero document_visibility rows (see CreateDocumentRequest).
|
||||
_, err = s.db.Exec(ctx, `
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS created_by BIGINT;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: alter documents add created_by: %w", err)
|
||||
}
|
||||
|
||||
// title_manually_set: tracks whether a user has explicitly renamed the
|
||||
// document (via PATCH .../title) versus the title still being whatever was
|
||||
// auto-derived from OCR text at upload time. Reprocess only re-derives the
|
||||
// title from fresh OCR when this is false, so a user's manual rename is
|
||||
// never silently overwritten. Defaults to false for all existing rows —
|
||||
// deliberate: we cannot distinguish old manual renames from old
|
||||
// auto-derived titles, and false is the safe choice that lets reprocess
|
||||
// improve stale/garbled auto-derived titles from before the OCR
|
||||
// rotation/deskew fix (see DEVLOG).
|
||||
_, err = s.db.Exec(ctx, `
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS title_manually_set BOOLEAN NOT NULL DEFAULT false;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: alter documents add title_manually_set: %w", err)
|
||||
}
|
||||
|
||||
// document_date: the belegdatum (invoice/document date) recognised from the
|
||||
// OCR text, separate from created_at (the immutable scan/upload timestamp).
|
||||
// Nullable — nil for existing rows and for documents where no plausible date
|
||||
// was found. Drives the store/<tenant>/<yyyy>/<mm>/ archival path at upload
|
||||
// time (see internal/api storeUploadedFile); reprocess refreshes the column
|
||||
// but never moves the already-archived WORM file. See
|
||||
// migrations/019_document_date.sql.
|
||||
_, err = s.db.Exec(ctx, `
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS document_date DATE;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: alter documents add document_date: %w", err)
|
||||
}
|
||||
|
||||
// document_date_score: confidence (0.4-0.9 automatic keyword-proximity
|
||||
// heuristic, 1.0 once a user manually confirms/overrides the date via PUT
|
||||
// .../document-date) for document_date. Nullable, no backfill for existing
|
||||
// rows — NULL means "score unknown", not "score 0". Reserved as a quality
|
||||
// gate for the future Buchhaltungs-Pull-API (only Score >= 0.75 pullable).
|
||||
// See migrations/025_document_date_score.sql.
|
||||
_, err = s.db.Exec(ctx, `
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS document_date_score NUMERIC;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: alter documents add document_date_score: %w", err)
|
||||
}
|
||||
|
||||
// Structured taxonomy entities (tags/document_types/correspondents) plus
|
||||
// the barcode-recognition columns on documents (see
|
||||
// migrations/005_taxonomy.sql / lazy-splashing-puppy plan).
|
||||
if err := s.initTaxonomySchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Custom fields (custom_field_defs / document_type_fields /
|
||||
// document_field_values — see migrations/006_custom_fields.sql).
|
||||
if err := s.initCustomFieldsSchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Papierkorb + gestaffeltes Löschkonzept (soft-delete + Vier-Augen-Prinzip
|
||||
// für finales Löschen — see migrations/007_trash.sql).
|
||||
if err := s.initTrashSchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Klassifizierungsvorlagen (classification templates — see
|
||||
// migrations/011_classification_templates.sql). Depends on document_types
|
||||
// (initTaxonomySchema) and custom_field_defs (initCustomFieldsSchema)
|
||||
// created above, so it is wired in AFTER them.
|
||||
if err := s.initClassificationTemplatesSchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Workflows / Consumption-Regeln (see migrations/012_workflows.sql). Wired
|
||||
// in AFTER classification templates because a workflow action can reference
|
||||
// a classification template (apply_classification_template).
|
||||
if err := s.initWorkflowsSchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Heuristische Metadaten-Vorschläge (metadata_suggestions — see
|
||||
// migrations/012_metadata_suggestions.sql). Wired in AFTER documents/taxonomy
|
||||
// exist, since suggestions score taxonomy entities against a document.
|
||||
if err := s.initMetadataSuggestionsSchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Pro-Mandant konfigurierbare Anbindung an einen EXTERNEN Ollama-Server
|
||||
// (tenant_ollama_config — see migrations/017_tenant_ollama_config.sql).
|
||||
// Gate für den optionalen 'ollama'-Provider der Metadaten-Vorschläge.
|
||||
if err := s.initOllamaConfigSchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Freitext-Notizen pro Dokument (document_notes — see
|
||||
// migrations/013_document_notes.sql). Wired in AFTER documents exists since
|
||||
// it FK-references documents(id).
|
||||
if err := s.initDocumentNotesSchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Gespeicherte Suchansichten (saved_views — SavedViews, Paperless-ngx
|
||||
// inspiriert). Tenant-/user-scoped, hängt an keiner anderen Tabelle, daher
|
||||
// zuletzt eingehängt.
|
||||
if err := s.initSavedViewsSchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// OCR-Wortpositionen (ocr_words — see migrations/024_ocr_words.sql), Phase
|
||||
// 2 of the OCR text-highlight/overlay feature. Wired in AFTER documents
|
||||
// exists since it FK-references documents(id) ON DELETE CASCADE.
|
||||
if err := s.initOCRWordsSchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Digitale Akten (digitaler Aktenordner — see
|
||||
// migrations/018_akten.sql / project_akte_konzept_plan.md). Wired in AFTER
|
||||
// documents + correspondents exist: initAktenSchema creates the akten table
|
||||
// (FK to correspondents) and then ALTERs documents to add akte_id (FK to
|
||||
// akten), so both referenced tables must already exist.
|
||||
if err := s.initAktenSchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// ML-Retraining-Klassifizierung Phase 1 (Naive-Bayes, ergänzt die
|
||||
// Regel-Engine — see migrations/020_ml_classifier.sql). Wired in AFTER
|
||||
// initTaxonomySchema since it FK-references nothing directly but ALTERs
|
||||
// documents/document_tags which must already exist.
|
||||
if err := s.initMLClassifierSchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// has_thumbnail: tracks whether an eager preview thumbnail was rendered at
|
||||
// upload/reprocess time (see storeUploadedFile/ReprocessDocument in
|
||||
// internal/api). Defaults to false for all existing rows — deliberate:
|
||||
// GET /api/documents/{id}/thumbnail already falls back to on-demand lazy
|
||||
// generation on a cache miss regardless of this flag (see
|
||||
// internal/thumbnail), so no backfill migration is needed here, this
|
||||
// column is purely a "was it pre-rendered" hint for the frontend.
|
||||
_, err = s.db.Exec(ctx, `
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS has_thumbnail BOOLEAN NOT NULL DEFAULT false;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: alter documents add has_thumbnail: %w", err)
|
||||
}
|
||||
|
||||
// GoBD-Aufbewahrungsregeln (retention rules / "Disposition Schedules" —
|
||||
// see migrations/022_retention_rules.sql). Wired in AFTER initTaxonomySchema
|
||||
// (FK to document_types) and AFTER initTrashSchema (retain_until/deleted_at
|
||||
// on documents must already exist — the rules engine only ever FEEDS
|
||||
// retain_until, the existing trash + Vier-Augen flow performs the actual
|
||||
// disposition).
|
||||
if err := s.initRetentionRulesSchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Mandanten-Job-Queue für die asynchrone Nachverarbeitung (processing_jobs
|
||||
// + documents.processing_status — see migrations/023_processing_jobs.sql).
|
||||
// Zuletzt eingehängt: die Tabelle FK-referenziert documents(id), das muss
|
||||
// also bereits existieren.
|
||||
if err := s.initProcessingJobsSchema(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateDocument inserts a new document and returns it.
|
||||
func (s *Store) CreateDocument(ctx context.Context, req CreateDocumentRequest) (*Document, error) {
|
||||
var d Document
|
||||
err := s.db.QueryRow(ctx, `
|
||||
INSERT INTO documents (tenant_id, title, doc_type, correspondent, storage_path, content_hash, ocr_text, retain_until, document_date, document_date_score, source, source_ref, created_by)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||
RETURNING id, tenant_id, title, COALESCE(doc_type, ''), COALESCE(correspondent, ''), doc_type_id, correspondent_id, storage_path, content_hash,
|
||||
COALESCE(ocr_text, ''), retain_until, document_date, document_date_score, COALESCE(source, ''), COALESCE(source_ref, ''), created_by, title_manually_set, has_thumbnail, processing_status, created_at, updated_at
|
||||
`, req.TenantID, req.Title, nullIfEmpty(req.DocType), nullIfEmpty(req.Correspondent), req.StoragePath, req.ContentHash,
|
||||
nullIfEmpty(req.OCRText), req.RetainUntil, req.DocumentDate, req.DocumentDateScore, nullIfEmpty(req.Source), nullIfEmpty(req.SourceRef), req.CreatedBy,
|
||||
).Scan(&d.ID, &d.TenantID, &d.Title, &d.DocType, &d.Correspondent, &d.DocTypeID, &d.CorrespondentID, &d.StoragePath, &d.ContentHash,
|
||||
&d.OCRText, &d.RetainUntil, &d.DocumentDate, &d.DocumentDateScore, &d.Source, &d.SourceRef, &d.CreatedBy, &d.TitleManuallySet, &d.HasThumbnail, &d.ProcessingStatus, &d.CreatedAt, &d.UpdatedAt)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return nil, ErrDuplicateContentHash
|
||||
}
|
||||
return nil, fmt.Errorf("storage: create document: %w", err)
|
||||
}
|
||||
return &d, nil
|
||||
}
|
||||
|
||||
// GetDocument retrieves a document by ID, scoped to tenant.
|
||||
func (s *Store) GetDocument(ctx context.Context, id, tenantID int64) (*Document, error) {
|
||||
var d Document
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT id, tenant_id, title, COALESCE(doc_type, ''), COALESCE(correspondent, ''), doc_type_id, correspondent_id, storage_path, content_hash,
|
||||
COALESCE(ocr_text, ''), retain_until, document_date, document_date_score, COALESCE(source, ''), COALESCE(source_ref, ''), created_by, title_manually_set, has_thumbnail, processing_status, created_at, updated_at
|
||||
FROM documents WHERE id = $1 AND tenant_id = $2
|
||||
`, id, tenantID).Scan(&d.ID, &d.TenantID, &d.Title, &d.DocType, &d.Correspondent, &d.DocTypeID, &d.CorrespondentID, &d.StoragePath, &d.ContentHash,
|
||||
&d.OCRText, &d.RetainUntil, &d.DocumentDate, &d.DocumentDateScore, &d.Source, &d.SourceRef, &d.CreatedBy, &d.TitleManuallySet, &d.HasThumbnail, &d.ProcessingStatus, &d.CreatedAt, &d.UpdatedAt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: get document: %w", err)
|
||||
}
|
||||
return &d, nil
|
||||
}
|
||||
|
||||
// DocumentExistsByHash reports whether the tenant already has a document with
|
||||
// the given content_hash. This mirrors the (tenant_id, content_hash) unique
|
||||
// index (deleted_at is intentionally NOT filtered, so the result matches what
|
||||
// CreateDocument's INSERT would enforce) and lets the upload pipeline detect a
|
||||
// duplicate BEFORE spending OCR/Tesseract cycles on the file.
|
||||
func (s *Store) DocumentExistsByHash(ctx context.Context, tenantID int64, contentHash string) (bool, error) {
|
||||
var exists bool
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT EXISTS (SELECT 1 FROM documents WHERE tenant_id = $1 AND content_hash = $2)
|
||||
`, tenantID, contentHash).Scan(&exists)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("storage: check document hash: %w", err)
|
||||
}
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
// ListDocuments returns all documents for a tenant, newest first.
|
||||
//
|
||||
// aclUserID applies the group-resolved document ACL: when non-nil, only
|
||||
// documents visible to that user via document_visibility (through their
|
||||
// permission group memberships) are returned. Callers pass a non-nil value
|
||||
// only for role 'user'; domain_admin/superadmin pass nil to see every document
|
||||
// in the tenant unfiltered (roles remain the outer boundary — see
|
||||
// permissions.go).
|
||||
func (s *Store) ListDocuments(ctx context.Context, tenantID int64, aclUserID *int64) ([]Document, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, tenant_id, title, COALESCE(doc_type, ''), COALESCE(correspondent, ''), doc_type_id, correspondent_id, storage_path, content_hash,
|
||||
COALESCE(ocr_text, ''), retain_until, document_date, document_date_score, COALESCE(source, ''), COALESCE(source_ref, ''), created_by, title_manually_set, has_thumbnail, processing_status, created_at, updated_at
|
||||
FROM documents
|
||||
WHERE tenant_id = $1 AND deleted_at IS NULL
|
||||
AND ($2::bigint IS NULL OR documents.created_by = $2 OR EXISTS (
|
||||
SELECT 1 FROM document_visibility dv
|
||||
JOIN permission_group_members pgm ON pgm.group_id = dv.group_id
|
||||
WHERE dv.document_id = documents.id AND pgm.user_id = $2
|
||||
))
|
||||
ORDER BY created_at DESC
|
||||
`, tenantID, aclUserID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list documents: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]Document, 0)
|
||||
for rows.Next() {
|
||||
var d Document
|
||||
if err := rows.Scan(&d.ID, &d.TenantID, &d.Title, &d.DocType, &d.Correspondent, &d.DocTypeID, &d.CorrespondentID, &d.StoragePath, &d.ContentHash,
|
||||
&d.OCRText, &d.RetainUntil, &d.DocumentDate, &d.DocumentDateScore, &d.Source, &d.SourceRef, &d.CreatedBy, &d.TitleManuallySet, &d.HasThumbnail, &d.ProcessingStatus, &d.CreatedAt, &d.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan document: %w", err)
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// IsDocumentVisible reports whether the given user may see the document, using
|
||||
// exactly the same rule as ListDocuments' ACL predicate (own upload OR a
|
||||
// document_visibility row resolved through the user's permission groups).
|
||||
// Always tenant-scoped: a document of another tenant is never visible, even to
|
||||
// its own uploader. Callers pass userID only for role 'user'; domain_admin /
|
||||
// superadmin skip this check (roles remain the outer ACL boundary).
|
||||
func (s *Store) IsDocumentVisible(ctx context.Context, id, tenantID, userID int64) (bool, error) {
|
||||
var visible bool
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM documents d
|
||||
WHERE d.id = $1 AND d.tenant_id = $2 AND d.deleted_at IS NULL
|
||||
AND (d.created_by = $3 OR EXISTS (
|
||||
SELECT 1 FROM document_visibility dv
|
||||
JOIN permission_group_members pgm ON pgm.group_id = dv.group_id
|
||||
WHERE dv.document_id = d.id AND pgm.user_id = $3
|
||||
))
|
||||
)
|
||||
`, id, tenantID, userID).Scan(&visible)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("storage: check document visibility: %w", err)
|
||||
}
|
||||
return visible, nil
|
||||
}
|
||||
|
||||
// DocumentTaxonomyNames resolves the structured document-type and correspondent
|
||||
// names of a document, tenant-scoped. Empty strings when unassigned. Used by
|
||||
// the single-document export (metadata.json) so the ZIP carries readable names
|
||||
// instead of raw IDs.
|
||||
func (s *Store) DocumentTaxonomyNames(ctx context.Context, id, tenantID int64) (docType, correspondent string, err error) {
|
||||
err = s.db.QueryRow(ctx, `
|
||||
SELECT COALESCE(dt.name, ''), COALESCE(c.name, '')
|
||||
FROM documents d
|
||||
LEFT JOIN document_types dt ON dt.id = d.doc_type_id AND dt.tenant_id = d.tenant_id
|
||||
LEFT JOIN correspondents c ON c.id = d.correspondent_id AND c.tenant_id = d.tenant_id
|
||||
WHERE d.id = $1 AND d.tenant_id = $2
|
||||
`, id, tenantID).Scan(&docType, &correspondent)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("storage: document taxonomy names: %w", err)
|
||||
}
|
||||
return docType, correspondent, nil
|
||||
}
|
||||
|
||||
// UpdateDocumentTitle renames a document, scoped to tenant, and marks the
|
||||
// title as manually set (title_manually_set = true). Use this for the
|
||||
// user-facing PATCH .../title endpoint only — it permanently opts the
|
||||
// document out of automatic title re-derivation by POST .../reprocess. Title
|
||||
// is not part of the ACL, so this only needs an index re-sync (mirrors
|
||||
// SetDocumentCorrespondent in taxonomy.go), not a visibility recompute.
|
||||
func (s *Store) UpdateDocumentTitle(ctx context.Context, id, tenantID int64, title string) error {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE documents SET title = $1, title_manually_set = true, updated_at = now() WHERE id = $2 AND tenant_id = $3`, title, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: update document title: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("storage: update document title: %w", ErrDocumentNotFound)
|
||||
}
|
||||
s.SyncIndex(ctx, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateDocumentTitleAuto renames a document without touching
|
||||
// title_manually_set. Used only by POST .../reprocess to re-derive a title
|
||||
// from freshly re-extracted OCR text — callers must already have checked
|
||||
// doc.TitleManuallySet is false before calling this, otherwise a user's
|
||||
// manual rename would be silently overwritten.
|
||||
func (s *Store) UpdateDocumentTitleAuto(ctx context.Context, id, tenantID int64, title string) error {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE documents SET title = $1, updated_at = now() WHERE id = $2 AND tenant_id = $3`, title, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: update document title (auto): %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("storage: update document title (auto): %w", ErrDocumentNotFound)
|
||||
}
|
||||
s.SyncIndex(ctx, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateDocumentOCRText replaces a document's ocr_text, scoped to tenant.
|
||||
// Used by the re-processing endpoint (POST /api/documents/{id}/reprocess)
|
||||
// after re-running OCR on an already-archived file — the WORM file itself is
|
||||
// never touched, only this derived metadata column is refreshed. OCR text is
|
||||
// not part of the ACL, so this only needs an index re-sync (mirrors
|
||||
// UpdateDocumentTitle), not a visibility recompute.
|
||||
func (s *Store) UpdateDocumentOCRText(ctx context.Context, id, tenantID int64, ocrText string) error {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE documents SET ocr_text = $1, updated_at = now() WHERE id = $2 AND tenant_id = $3`, nullIfEmpty(ocrText), id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: update document ocr_text: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("storage: update document ocr_text: %w", ErrDocumentNotFound)
|
||||
}
|
||||
s.SyncIndex(ctx, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateDocumentDate replaces a document's recognised belegdatum
|
||||
// (document_date) and its confidence (document_date_score), scoped to tenant.
|
||||
// Used by the re-processing endpoint after re-running OCR so the metadata
|
||||
// reflects a freshly recognised date — the WORM file and its
|
||||
// store/<yyyy>/<mm>/ path are NEVER moved, only this derived column is
|
||||
// refreshed. Passing a nil date clears both date and score. score should be
|
||||
// 1.0 when the caller is a manual user confirmation (PUT .../document-date),
|
||||
// or the heuristic's own confidence (0.4-0.9) for automatic
|
||||
// extract/reprocess/job-queue callers — never carry over a stale automatic
|
||||
// score after a manual override. document_date is not part of the ACL, so
|
||||
// this only needs an index re-sync (mirrors UpdateDocumentOCRText), not a
|
||||
// visibility recompute.
|
||||
func (s *Store) UpdateDocumentDate(ctx context.Context, id, tenantID int64, date *time.Time, score *float64) error {
|
||||
if date == nil {
|
||||
score = nil
|
||||
}
|
||||
tag, err := s.db.Exec(ctx, `UPDATE documents SET document_date = $1, document_date_score = $2, updated_at = now() WHERE id = $3 AND tenant_id = $4`, date, score, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: update document date: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("storage: update document date: %w", ErrDocumentNotFound)
|
||||
}
|
||||
s.SyncIndex(ctx, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetDocumentHasThumbnail records whether an eager preview thumbnail was
|
||||
// successfully rendered for this document, scoped to tenant. Used by
|
||||
// storeUploadedFile (after upload) and ReprocessDocument (to backfill/repair
|
||||
// a previously missing thumbnail) — see internal/api/document_handlers.go.
|
||||
// has_thumbnail is a pure UI hint (not indexed content, not part of the ACL),
|
||||
// so unlike UpdateDocumentTitle/OCRText/Date this deliberately does NOT call
|
||||
// SyncIndex — no Manticore reindex is needed for a thumbnail flag.
|
||||
func (s *Store) SetDocumentHasThumbnail(ctx context.Context, id, tenantID int64, hasThumbnail bool) error {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE documents SET has_thumbnail = $1, updated_at = now() WHERE id = $2 AND tenant_id = $3`, hasThumbnail, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: set document has_thumbnail: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("storage: set document has_thumbnail: %w", ErrDocumentNotFound)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteDocument removes a document, scoped to tenant.
|
||||
func (s *Store) DeleteDocument(ctx context.Context, id, tenantID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `DELETE FROM documents WHERE id = $1 AND tenant_id = $2`, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: delete document: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("storage: document not found or not owned by tenant")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func nullIfEmpty(s string) any {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// nullIfEmptyPtr maps a *string to a NULL-able SQL argument: nil or a
|
||||
// whitespace-only value becomes NULL, otherwise the trimmed string is stored.
|
||||
func nullIfEmptyPtr(s *string) any {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(*s) == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.TrimSpace(*s)
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"archivdms/internal/index"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// Full-text search index sync (Phase 1, see internal/index).
|
||||
//
|
||||
// Postgres is the single source of truth; these helpers keep the secondary
|
||||
// per-tenant Manticore index in step. They are strictly best-effort: an index
|
||||
// failure is logged and swallowed, NEVER returned to the caller, so a search
|
||||
// backend hiccup can never block or fail a document write. When no indexer is
|
||||
// configured (s.indexer == nil) every helper is a no-op.
|
||||
|
||||
// SyncIndex re-projects a document (with its tags + resolved ACL groups) into
|
||||
// the search index. Safe to call after any change that affects an indexed
|
||||
// field: create, tag attach/detach, doc_type/correspondent change, custom
|
||||
// fields, or an ACL recompute. Best-effort — errors are logged, not returned.
|
||||
func (s *Store) SyncIndex(ctx context.Context, documentID int64) {
|
||||
if s.indexer == nil {
|
||||
return
|
||||
}
|
||||
doc, err := s.buildDocumentDoc(ctx, documentID)
|
||||
if err != nil {
|
||||
s.logIndexWarn("build index doc", documentID, err)
|
||||
return
|
||||
}
|
||||
if doc == nil {
|
||||
// Row vanished (or tombstoned) — treat as a delete.
|
||||
return
|
||||
}
|
||||
if err := s.indexer.ForTenant(doc.TenantID).IndexSync(ctx, *doc); err != nil {
|
||||
s.logIndexWarn("index sync", documentID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteFromIndex removes a document from the search index. Used on final
|
||||
// (executed) deletion — GoBD-critical: a purged document must not remain
|
||||
// findable. Best-effort — errors are logged, not returned. tenantID is passed
|
||||
// explicitly because the DB row may already be gone/tombstoned.
|
||||
func (s *Store) DeleteFromIndex(ctx context.Context, documentID, tenantID int64) {
|
||||
if s.indexer == nil {
|
||||
return
|
||||
}
|
||||
if err := s.indexer.ForTenant(tenantID).Delete(ctx, documentID); err != nil {
|
||||
s.logIndexWarn("index delete", documentID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// buildDocumentDoc assembles the index projection for a single document from
|
||||
// the documents row plus its tags (document_tags/tags) and resolved ACL groups
|
||||
// (document_visibility). Returns (nil, nil) if the document does not exist.
|
||||
func (s *Store) buildDocumentDoc(ctx context.Context, documentID int64) (*index.DocumentDoc, error) {
|
||||
var d index.DocumentDoc
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT id, tenant_id, title, COALESCE(doc_type, ''), COALESCE(correspondent, ''),
|
||||
doc_type_id, correspondent_id, COALESCE(ocr_text, ''), retain_until, created_at, updated_at
|
||||
FROM documents WHERE id = $1
|
||||
`, documentID).Scan(&d.ID, &d.TenantID, &d.Title, &d.DocType, &d.Correspondent,
|
||||
&d.DocTypeID, &d.CorrespondentID, &d.OCRText, &d.RetainUntil, &d.CreatedAt, &d.UpdatedAt)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Tags (names + ids), tenant-scoped via the join to tags.
|
||||
tagRows, err := s.db.Query(ctx, `
|
||||
SELECT t.id, t.name FROM tags t
|
||||
JOIN document_tags dt ON dt.tag_id = t.id
|
||||
WHERE dt.document_id = $1 AND t.tenant_id = $2
|
||||
ORDER BY t.name ASC
|
||||
`, documentID, d.TenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for tagRows.Next() {
|
||||
var id int64
|
||||
var name string
|
||||
if err := tagRows.Scan(&id, &name); err != nil {
|
||||
tagRows.Close()
|
||||
return nil, err
|
||||
}
|
||||
d.TagIDs = append(d.TagIDs, id)
|
||||
d.Tags = append(d.Tags, name)
|
||||
}
|
||||
tagRows.Close()
|
||||
if err := tagRows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Resolved ACL groups (materialised visibility).
|
||||
aclRows, err := s.db.Query(ctx, `
|
||||
SELECT group_id FROM document_visibility WHERE document_id = $1 ORDER BY group_id
|
||||
`, documentID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for aclRows.Next() {
|
||||
var gid int64
|
||||
if err := aclRows.Scan(&gid); err != nil {
|
||||
aclRows.Close()
|
||||
return nil, err
|
||||
}
|
||||
d.ACLGroupIDs = append(d.ACLGroupIDs, gid)
|
||||
}
|
||||
aclRows.Close()
|
||||
if err := aclRows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &d, nil
|
||||
}
|
||||
|
||||
// ErrNoIndexer is returned by reindex helpers when no search index is wired
|
||||
// into the store. Unlike the request-path sync helpers (which degrade to a
|
||||
// silent no-op when s.indexer == nil), an explicit reindex must fail loudly so
|
||||
// an operator never mistakes a no-op for a successful rebuild.
|
||||
var ErrNoIndexer = errors.New("storage: no search index configured")
|
||||
|
||||
// ReindexTenant rebuilds the full-text search index for a single tenant from
|
||||
// Postgres (the source of truth). It streams all non-deleted documents of the
|
||||
// tenant in ascending-id batches (keyset pagination, batchSize rows at a time)
|
||||
// so memory stays bounded even for very large tenants, projects each via
|
||||
// buildDocumentDoc and upserts it through the tenant's Indexer.
|
||||
//
|
||||
// progress, if non-nil, is invoked after each successfully indexed document
|
||||
// with (done, total) so callers can log progress. Returns the number of
|
||||
// documents indexed. Fails with ErrNoIndexer when no indexer is configured.
|
||||
func (s *Store) ReindexTenant(ctx context.Context, tenantID int64, batchSize int, progress func(done, total int)) (int, error) {
|
||||
if s.indexer == nil {
|
||||
return 0, ErrNoIndexer
|
||||
}
|
||||
if batchSize <= 0 {
|
||||
batchSize = 500
|
||||
}
|
||||
|
||||
var total int
|
||||
if err := s.db.QueryRow(ctx,
|
||||
`SELECT COUNT(*) FROM documents WHERE tenant_id = $1 AND deleted_at IS NULL`,
|
||||
tenantID,
|
||||
).Scan(&total); err != nil {
|
||||
return 0, fmt.Errorf("storage: reindex count tenant %d: %w", tenantID, err)
|
||||
}
|
||||
|
||||
indexer := s.indexer.ForTenant(tenantID)
|
||||
|
||||
done := 0
|
||||
var lastID int64
|
||||
for {
|
||||
ids, err := s.reindexDocumentIDs(ctx, tenantID, lastID, batchSize)
|
||||
if err != nil {
|
||||
return done, err
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
break
|
||||
}
|
||||
for _, id := range ids {
|
||||
doc, err := s.buildDocumentDoc(ctx, id)
|
||||
if err != nil {
|
||||
return done, fmt.Errorf("storage: reindex build doc id=%d: %w", id, err)
|
||||
}
|
||||
if doc == nil {
|
||||
// Row vanished/tombstoned between the id scan and now — skip.
|
||||
continue
|
||||
}
|
||||
if err := indexer.IndexSync(ctx, *doc); err != nil {
|
||||
return done, fmt.Errorf("storage: reindex index doc id=%d: %w", id, err)
|
||||
}
|
||||
done++
|
||||
if progress != nil {
|
||||
progress(done, total)
|
||||
}
|
||||
}
|
||||
lastID = ids[len(ids)-1]
|
||||
}
|
||||
|
||||
return done, nil
|
||||
}
|
||||
|
||||
// reindexDocumentIDs returns up to limit non-deleted document IDs for a tenant
|
||||
// with id > afterID, ascending (keyset pagination).
|
||||
func (s *Store) reindexDocumentIDs(ctx context.Context, tenantID, afterID int64, limit int) ([]int64, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id FROM documents
|
||||
WHERE tenant_id = $1 AND deleted_at IS NULL AND id > $2
|
||||
ORDER BY id ASC
|
||||
LIMIT $3
|
||||
`, tenantID, afterID, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: reindex list ids tenant %d: %w", tenantID, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var ids []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("storage: reindex scan id: %w", err)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) logIndexWarn(op string, documentID int64, err error) {
|
||||
if s.logger != nil {
|
||||
s.logger.Warn("index sync failed", "op", op, "document_id", documentID, "err", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"archivdms/internal/matching"
|
||||
)
|
||||
|
||||
// ErrSuggestionNotFound is returned when no metadata_suggestions row exists
|
||||
// for a document (GetLatestSuggestion) or the id/tenant scope does not match
|
||||
// (MarkSuggestionReviewed).
|
||||
var ErrSuggestionNotFound = errors.New("storage: metadata suggestion not found")
|
||||
|
||||
// suggestionFloor is the minimum fuzzy score (0..1) at which a
|
||||
// tag/document_type/correspondent is surfaced as a NON-binding suggestion.
|
||||
// It sits deliberately BELOW matching.FuzzyThreshold (currently 0.85, the
|
||||
// auto-assign confidence): candidates at/above FuzzyThreshold that were not
|
||||
// already auto-assigned (e.g. a second document_type that also matched, or a
|
||||
// name-only near-match on an entity whose configured algorithm isn't fuzzy)
|
||||
// are strong suggestions; candidates in [suggestionFloor, FuzzyThreshold) are
|
||||
// the near-misses this feature exists to expose for manual review. 0.55 keeps
|
||||
// noise low while still catching typo-level OCR differences.
|
||||
const suggestionFloor = 0.55
|
||||
|
||||
// maxSuggestionCandidates caps how many candidates are kept per category
|
||||
// (tags / document_types / correspondents), sorted by score descending.
|
||||
const maxSuggestionCandidates = 5
|
||||
|
||||
// autoGeneratedTitlePattern matches the timestamp placeholder title produced
|
||||
// by titleFromOCRText ("Scan DD.MM.YYYY HH:MM") when no meaningful heading
|
||||
// could be derived on ingest. A current title matching this is treated as
|
||||
// "not a real title yet", so a re-derived title is suggested.
|
||||
var autoGeneratedTitlePattern = regexp.MustCompile(`^Scan \d{2}\.\d{2}\.\d{4} \d{2}:\d{2}$`)
|
||||
|
||||
// SuggestionCandidate is one scored, non-binding metadata suggestion for a
|
||||
// single taxonomy entity. Score is the fuzzy similarity in [0,1].
|
||||
type SuggestionCandidate struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Score float64 `json:"score"`
|
||||
Explanation []string `json:"explanation,omitempty"`
|
||||
}
|
||||
|
||||
// DocumentDateCandidate is a non-binding belegdatum (invoice/document date)
|
||||
// suggestion re-derived from the OCR text. Date is the ISO date (YYYY-MM-DD)
|
||||
// the frontend can apply via PUT /api/documents/{id}/document-date; Score is a
|
||||
// fixed heuristic confidence (the regex date scanner has no per-match score).
|
||||
type DocumentDateCandidate struct {
|
||||
Date string `json:"date"`
|
||||
Score float64 `json:"score"`
|
||||
}
|
||||
|
||||
// SuggestionPayload is the JSONB body persisted in metadata_suggestions.suggestion.
|
||||
// A nil Title means no title suggestion was made (current title already looks
|
||||
// human-authored). The candidate slices are always non-nil (possibly empty).
|
||||
// DocumentDateCandidate is nil when the document already has a belegdatum set or
|
||||
// no plausible date could be recognised in the OCR text.
|
||||
type SuggestionPayload struct {
|
||||
Title *string `json:"title,omitempty"`
|
||||
DocTypeCandidates []SuggestionCandidate `json:"doc_type_candidates"`
|
||||
CorrespondentCandidates []SuggestionCandidate `json:"correspondent_candidates"`
|
||||
TagCandidates []SuggestionCandidate `json:"tag_candidates"`
|
||||
DocumentDateCandidate *DocumentDateCandidate `json:"document_date_candidate,omitempty"`
|
||||
}
|
||||
|
||||
// MetadataSuggestion is one persisted suggestion run for a document. It is a
|
||||
// log/cache of what the heuristic provider proposed — applying an accepted
|
||||
// field goes through the normal edit endpoints, NOT through this row.
|
||||
type MetadataSuggestion struct {
|
||||
ID int64 `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
DocumentID int64 `json:"document_id"`
|
||||
Provider string `json:"provider"`
|
||||
RequestedBy *int64 `json:"requested_by,omitempty"`
|
||||
RequestedAt time.Time `json:"requested_at"`
|
||||
Suggestion SuggestionPayload `json:"suggestion"`
|
||||
Status string `json:"status"`
|
||||
ReviewedAt *time.Time `json:"reviewed_at,omitempty"`
|
||||
ReviewedBy *int64 `json:"reviewed_by,omitempty"`
|
||||
}
|
||||
|
||||
// initMetadataSuggestionsSchema creates the metadata_suggestions table.
|
||||
// Idempotent, called from (*Store).initSchema AFTER the documents/taxonomy
|
||||
// schema exists. Documented (not executed) in
|
||||
// migrations/012_metadata_suggestions.sql.
|
||||
func (s *Store) initMetadataSuggestionsSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS metadata_suggestions (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
document_id BIGINT NOT NULL,
|
||||
provider TEXT NOT NULL DEFAULT 'heuristic',
|
||||
requested_by BIGINT,
|
||||
requested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
suggestion JSONB NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','reviewed')),
|
||||
reviewed_at TIMESTAMPTZ,
|
||||
reviewed_by BIGINT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_metadata_suggestions_document ON metadata_suggestions(document_id);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create metadata suggestions table: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const metadataSuggestionCols = `id, tenant_id, document_id, provider, requested_by, requested_at, suggestion, status, reviewed_at, reviewed_by`
|
||||
|
||||
func scanMetadataSuggestion(row interface {
|
||||
Scan(dest ...any) error
|
||||
}) (*MetadataSuggestion, error) {
|
||||
var m MetadataSuggestion
|
||||
var payload []byte
|
||||
if err := row.Scan(&m.ID, &m.TenantID, &m.DocumentID, &m.Provider, &m.RequestedBy,
|
||||
&m.RequestedAt, &payload, &m.Status, &m.ReviewedAt, &m.ReviewedBy); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Suggestion = SuggestionPayload{
|
||||
DocTypeCandidates: make([]SuggestionCandidate, 0),
|
||||
CorrespondentCandidates: make([]SuggestionCandidate, 0),
|
||||
TagCandidates: make([]SuggestionCandidate, 0),
|
||||
}
|
||||
if len(payload) > 0 {
|
||||
if err := json.Unmarshal(payload, &m.Suggestion); err != nil {
|
||||
return nil, fmt.Errorf("storage: unmarshal suggestion payload: %w", err)
|
||||
}
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
// GenerateHeuristicSuggestions builds a fresh, rule-based (no LLM) metadata
|
||||
// suggestion for a document: it fuzzy-scores every taxonomy entity's name (and
|
||||
// its configured match_pattern, if any) against the document's title+OCR text,
|
||||
// surfaces the near-misses above suggestionFloor that are NOT already assigned,
|
||||
// and — if the current title still looks auto-generated — proposes a
|
||||
// re-derived title. The result is persisted as a metadata_suggestions row and
|
||||
// returned. requestedBy may be nil for non-interactive callers.
|
||||
func (s *Store) GenerateHeuristicSuggestions(ctx context.Context, documentID, tenantID int64, requestedBy *int64) (*MetadataSuggestion, error) {
|
||||
doc, err := s.GetDocument(ctx, documentID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err // ErrDocumentNotFound propagates
|
||||
}
|
||||
|
||||
haystack := doc.Title
|
||||
if doc.OCRText != "" {
|
||||
haystack = doc.Title + "\n" + doc.OCRText
|
||||
}
|
||||
|
||||
// Entities already assigned to the document — excluded from suggestions.
|
||||
assignedTags := map[int64]bool{}
|
||||
tags, err := s.ListDocumentTags(ctx, documentID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, t := range tags {
|
||||
assignedTags[t.ID] = true
|
||||
}
|
||||
|
||||
tagCands, err := s.scoreCandidates(ctx, "tags", tenantID, haystack, func(id int64) bool { return assignedTags[id] })
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
docTypeCands, err := s.scoreCandidates(ctx, "document_types", tenantID, haystack, func(id int64) bool {
|
||||
return doc.DocTypeID != nil && *doc.DocTypeID == id
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
corrCands, err := s.scoreCandidates(ctx, "correspondents", tenantID, haystack, func(id int64) bool {
|
||||
return doc.CorrespondentID != nil && *doc.CorrespondentID == id
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
payload := SuggestionPayload{
|
||||
DocTypeCandidates: docTypeCands,
|
||||
CorrespondentCandidates: corrCands,
|
||||
TagCandidates: tagCands,
|
||||
}
|
||||
if autoGeneratedTitlePattern.MatchString(strings.TrimSpace(doc.Title)) {
|
||||
if t := heuristicTitle(doc.OCRText); t != "" && t != doc.Title {
|
||||
payload.Title = &t
|
||||
}
|
||||
}
|
||||
// Belegdatum suggestion: only when the document has no document_date yet and
|
||||
// a plausible date is recognisable in the OCR text. Surfaced as a chip the
|
||||
// user can apply via PUT /api/documents/{id}/document-date. The confidence is
|
||||
// now derived from keyword-proximity scoring (see documentDateFromTextWithScore)
|
||||
// instead of a fixed value.
|
||||
if doc.DocumentDate == nil {
|
||||
if d, sc, ok := documentDateFromTextWithScore(doc.OCRText); ok {
|
||||
payload.DocumentDateCandidate = &DocumentDateCandidate{
|
||||
Date: d.Format("2006-01-02"),
|
||||
Score: sc,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
raw, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: marshal suggestion payload: %w", err)
|
||||
}
|
||||
|
||||
row := s.db.QueryRow(ctx, `
|
||||
INSERT INTO metadata_suggestions (tenant_id, document_id, provider, requested_by, suggestion)
|
||||
VALUES ($1, $2, 'heuristic', $3, $4)
|
||||
RETURNING `+metadataSuggestionCols,
|
||||
tenantID, documentID, requestedBy, raw)
|
||||
m, err := scanMetadataSuggestion(row)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: insert metadata suggestion: %w", err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// scoreCandidates fuzzy-scores every entity of a kind for a tenant against the
|
||||
// haystack, keeps those at/above suggestionFloor that are not excluded (already
|
||||
// assigned), sorts by score descending and caps at maxSuggestionCandidates.
|
||||
func (s *Store) scoreCandidates(ctx context.Context, kind string, tenantID int64, haystack string, excluded func(id int64) bool) ([]SuggestionCandidate, error) {
|
||||
entities, err := s.ListTaxonomyEntities(ctx, kind, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]SuggestionCandidate, 0)
|
||||
for _, e := range entities {
|
||||
if excluded(e.ID) {
|
||||
continue
|
||||
}
|
||||
score := matching.FuzzyScore(e.Name, e.CaseSensitive, haystack)
|
||||
if e.MatchPattern != "" {
|
||||
if p := matching.FuzzyScore(e.MatchPattern, e.CaseSensitive, haystack); p > score {
|
||||
score = p
|
||||
}
|
||||
}
|
||||
if score < suggestionFloor {
|
||||
continue
|
||||
}
|
||||
out = append(out, SuggestionCandidate{ID: e.ID, Name: e.Name, Score: score})
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Score > out[j].Score })
|
||||
if len(out) > maxSuggestionCandidates {
|
||||
out = out[:maxSuggestionCandidates]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// heuristicTitle re-derives a candidate title from OCR text using a different
|
||||
// heuristic than titleFromOCRText (which takes the first meaningful line): it
|
||||
// picks the longest trimmed line, which is more likely to be a real heading /
|
||||
// company line than a short letterhead fragment or page number. Returns "" if
|
||||
// no line qualifies. Kept simple on purpose (no NLP).
|
||||
func heuristicTitle(ocrText string) string {
|
||||
if ocrText == "" {
|
||||
return ""
|
||||
}
|
||||
best := ""
|
||||
bestLen := 0
|
||||
for _, line := range strings.Split(ocrText, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
r := []rune(line)
|
||||
if len(r) < 5 {
|
||||
continue
|
||||
}
|
||||
if len(r) > bestLen {
|
||||
bestLen = len(r)
|
||||
if len(r) > 120 {
|
||||
line = string(r[:120])
|
||||
}
|
||||
best = line
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// GetLatestSuggestion returns the most recent metadata_suggestions row for a
|
||||
// document, scoped to tenant ownership, or ErrSuggestionNotFound if none exist.
|
||||
func (s *Store) GetLatestSuggestion(ctx context.Context, documentID, tenantID int64) (*MetadataSuggestion, error) {
|
||||
row := s.db.QueryRow(ctx, `SELECT `+metadataSuggestionCols+`
|
||||
FROM metadata_suggestions
|
||||
WHERE document_id = $1 AND tenant_id = $2
|
||||
ORDER BY requested_at DESC, id DESC
|
||||
LIMIT 1`, documentID, tenantID)
|
||||
m, err := scanMetadataSuggestion(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrSuggestionNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("storage: get latest metadata suggestion: %w", err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// MarkSuggestionReviewed flags a suggestion row as reviewed (the user has acted
|
||||
// on it in the UI, regardless of which fields they accepted — those went
|
||||
// through the normal edit endpoints). Scoped to tenant ownership. Returns
|
||||
// ErrSuggestionNotFound if the id/tenant scope does not match.
|
||||
func (s *Store) MarkSuggestionReviewed(ctx context.Context, id, tenantID, reviewedBy int64) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE metadata_suggestions
|
||||
SET status = 'reviewed', reviewed_at = now(), reviewed_by = $1
|
||||
WHERE id = $2 AND tenant_id = $3`, reviewedBy, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: mark metadata suggestion reviewed: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrSuggestionNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"archivdms/internal/classifier"
|
||||
)
|
||||
|
||||
// GenerateNaiveBayesSuggestions builds a metadata suggestion for a document
|
||||
// using the trained Naive-Bayes model (internal/classifier) instead of the
|
||||
// fuzzy-name heuristic or an LLM. It classifies the document's title+OCR text
|
||||
// against the tenant's trained document_types / correspondents / tags models,
|
||||
// maps the predicted class IDs back to taxonomy entities, drops entities that
|
||||
// are already assigned, and persists the result as a metadata_suggestions row
|
||||
// with provider='naive_bayes' — in the SAME SuggestionPayload schema the other
|
||||
// providers produce, so the API/frontend are unchanged.
|
||||
//
|
||||
// A kind whose model is untrained (or below the per-class data threshold) simply
|
||||
// yields no candidates for that kind — not an error. Any real failure (DB error,
|
||||
// classifier error) is returned as-is: there is NO silent fallback to the
|
||||
// heuristic provider (GoBD-Nachvollziehbarkeit — the caller reports which
|
||||
// provider produced or failed the run). requestedBy may be nil for
|
||||
// non-interactive callers.
|
||||
func (s *Store) GenerateNaiveBayesSuggestions(ctx context.Context, documentID, tenantID int64, requestedBy *int64) (*MetadataSuggestion, error) {
|
||||
doc, err := s.GetDocument(ctx, documentID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err // ErrDocumentNotFound propagates
|
||||
}
|
||||
|
||||
text := doc.Title
|
||||
if doc.OCRText != "" {
|
||||
text = doc.Title + "\n" + doc.OCRText
|
||||
}
|
||||
|
||||
clf := classifier.New(s.db)
|
||||
|
||||
// Entities already assigned are excluded from suggestions, matching the
|
||||
// other providers' behaviour.
|
||||
assignedTags := map[int64]bool{}
|
||||
docTags, err := s.ListDocumentTags(ctx, documentID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, t := range docTags {
|
||||
assignedTags[t.ID] = true
|
||||
}
|
||||
|
||||
docTypeCands, err := s.naiveBayesCandidates(ctx, clf, "document_types", tenantID, text, func(id int64) bool {
|
||||
return doc.DocTypeID != nil && *doc.DocTypeID == id
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
corrCands, err := s.naiveBayesCandidates(ctx, clf, "correspondents", tenantID, text, func(id int64) bool {
|
||||
return doc.CorrespondentID != nil && *doc.CorrespondentID == id
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tagCands, err := s.naiveBayesCandidates(ctx, clf, "tags", tenantID, text, func(id int64) bool {
|
||||
return assignedTags[id]
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
payload := SuggestionPayload{
|
||||
DocTypeCandidates: docTypeCands,
|
||||
CorrespondentCandidates: corrCands,
|
||||
TagCandidates: tagCands,
|
||||
}
|
||||
// The Naive-Bayes model does not propose a title (it classifies against
|
||||
// existing entities only); Title stays nil.
|
||||
|
||||
raw, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: marshal naive_bayes suggestion payload: %w", err)
|
||||
}
|
||||
|
||||
row := s.db.QueryRow(ctx, `
|
||||
INSERT INTO metadata_suggestions (tenant_id, document_id, provider, requested_by, suggestion)
|
||||
VALUES ($1, $2, 'naive_bayes', $3, $4)
|
||||
RETURNING `+metadataSuggestionCols,
|
||||
tenantID, documentID, requestedBy, raw)
|
||||
m, err := scanMetadataSuggestion(row)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: insert naive_bayes metadata suggestion: %w", err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// naiveBayesCandidates runs the classifier for one kind and maps predicted class
|
||||
// IDs back to SuggestionCandidate (resolving the entity name from the taxonomy),
|
||||
// dropping excluded (already-assigned) entities and any predicted ID that no
|
||||
// longer exists as a live entity. Result is always non-nil.
|
||||
func (s *Store) naiveBayesCandidates(ctx context.Context, clf *classifier.Classifier, kind string, tenantID int64, text string, excluded func(id int64) bool) ([]SuggestionCandidate, error) {
|
||||
preds, err := clf.Predict(ctx, tenantID, kind, text)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: naive_bayes predict %s: %w", kind, err)
|
||||
}
|
||||
if len(preds) == 0 {
|
||||
return make([]SuggestionCandidate, 0), nil
|
||||
}
|
||||
|
||||
entities, err := s.ListTaxonomyEntities(ctx, kind, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := make(map[int64]string, len(entities))
|
||||
for _, e := range entities {
|
||||
names[e.ID] = e.Name
|
||||
}
|
||||
|
||||
out := make([]SuggestionCandidate, 0, len(preds))
|
||||
for _, p := range preds {
|
||||
if excluded(p.EntityID) {
|
||||
continue
|
||||
}
|
||||
name, ok := names[p.EntityID]
|
||||
if !ok {
|
||||
continue // predicted a class whose entity was deleted since training
|
||||
}
|
||||
out = append(out, SuggestionCandidate{ID: p.EntityID, Name: name, Score: p.Score, Explanation: p.TopTokens})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archivdms/internal/llm"
|
||||
"archivdms/internal/matching"
|
||||
)
|
||||
|
||||
// maxOllamaOCRChars caps how much OCR text is fed into the prompt. The target
|
||||
// model is small (qwen2.5:1.5b, ~4GB-RAM server) with a limited context window,
|
||||
// so the first chunk of the document plus the taxonomy lists is all it sees.
|
||||
const maxOllamaOCRChars = 2000
|
||||
|
||||
// ollamaNameMatchFloor is the minimum fuzzy score at which an LLM-returned name
|
||||
// is accepted as referring to an existing taxonomy entity. The LLM only knows
|
||||
// names, never IDs, so its answers are mapped back to entities by name; anything
|
||||
// below this is treated as a hallucinated / non-existent entity and dropped.
|
||||
const ollamaNameMatchFloor = 0.8
|
||||
|
||||
// ollamaSuggestionResponse is the JSON schema the model is asked to fill. It
|
||||
// deliberately uses plain name lists (not the ID-bearing SuggestionCandidate
|
||||
// shape) because the LLM has no knowledge of internal IDs — names are mapped
|
||||
// back to entities afterwards.
|
||||
type ollamaSuggestionResponse struct {
|
||||
Title string `json:"title"`
|
||||
DocTypes []string `json:"doc_types"`
|
||||
Correspondents []string `json:"correspondents"`
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
|
||||
// GenerateOllamaSuggestions asks an EXTERNAL Ollama server (per-tenant config)
|
||||
// to propose metadata for a document and persists the result as a
|
||||
// metadata_suggestions row with provider='ollama', in the SAME SuggestionPayload
|
||||
// schema the heuristic provider produces (so the API/frontend are unchanged).
|
||||
//
|
||||
// The prompt contains the document title, a truncated slice of its OCR text and
|
||||
// the tenant's existing tags/document_types/correspondents (by name) so the
|
||||
// model reuses known entities. The model returns names; those are mapped back to
|
||||
// entity IDs by exact-then-fuzzy name match. Any Ollama error (unreachable,
|
||||
// timeout, invalid JSON) is returned as-is — NO silent fallback to heuristic
|
||||
// (GoBD-Nachvollziehbarkeit: the caller reports which provider failed).
|
||||
func (s *Store) GenerateOllamaSuggestions(ctx context.Context, documentID, tenantID int64, requestedBy *int64, cfg OllamaConfig) (*MetadataSuggestion, error) {
|
||||
if !cfg.Enabled {
|
||||
return nil, fmt.Errorf("storage: ollama provider not enabled for tenant")
|
||||
}
|
||||
|
||||
doc, err := s.GetDocument(ctx, documentID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err // ErrDocumentNotFound propagates
|
||||
}
|
||||
|
||||
tags, err := s.ListTaxonomyEntities(ctx, "tags", tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
docTypes, err := s.ListTaxonomyEntities(ctx, "document_types", tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
correspondents, err := s.ListTaxonomyEntities(ctx, "correspondents", tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prompt := buildOllamaPrompt(doc, tags, docTypes, correspondents)
|
||||
|
||||
raw, err := llm.GenerateJSON(ctx, cfg.BaseURL, cfg.Model, time.Duration(cfg.TimeoutSeconds)*time.Second, prompt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: ollama generate: %w", err)
|
||||
}
|
||||
|
||||
var parsed ollamaSuggestionResponse
|
||||
if err := json.Unmarshal(raw, &parsed); err != nil {
|
||||
return nil, fmt.Errorf("storage: ollama response does not match expected schema: %w", err)
|
||||
}
|
||||
|
||||
// Entities already assigned are excluded from the suggestions, matching the
|
||||
// heuristic provider's behaviour.
|
||||
assignedTags := map[int64]bool{}
|
||||
docTags, err := s.ListDocumentTags(ctx, documentID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, t := range docTags {
|
||||
assignedTags[t.ID] = true
|
||||
}
|
||||
|
||||
payload := SuggestionPayload{
|
||||
DocTypeCandidates: mapNamesToCandidates(parsed.DocTypes, docTypes, func(id int64) bool {
|
||||
return doc.DocTypeID != nil && *doc.DocTypeID == id
|
||||
}),
|
||||
CorrespondentCandidates: mapNamesToCandidates(parsed.Correspondents, correspondents, func(id int64) bool {
|
||||
return doc.CorrespondentID != nil && *doc.CorrespondentID == id
|
||||
}),
|
||||
TagCandidates: mapNamesToCandidates(parsed.Tags, tags, func(id int64) bool { return assignedTags[id] }),
|
||||
}
|
||||
if t := strings.TrimSpace(parsed.Title); t != "" && t != doc.Title {
|
||||
payload.Title = &t
|
||||
}
|
||||
|
||||
rawPayload, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: marshal ollama suggestion payload: %w", err)
|
||||
}
|
||||
|
||||
row := s.db.QueryRow(ctx, `
|
||||
INSERT INTO metadata_suggestions (tenant_id, document_id, provider, requested_by, suggestion)
|
||||
VALUES ($1, $2, 'ollama', $3, $4)
|
||||
RETURNING `+metadataSuggestionCols,
|
||||
tenantID, documentID, requestedBy, rawPayload)
|
||||
m, err := scanMetadataSuggestion(row)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: insert ollama metadata suggestion: %w", err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// buildOllamaPrompt assembles a strict, schema-forcing prompt. Small models
|
||||
// need the format spelled out explicitly and benefit from being told to only
|
||||
// pick from the provided lists.
|
||||
func buildOllamaPrompt(doc *Document, tags, docTypes, correspondents []TaxonomyEntity) string {
|
||||
ocr := doc.OCRText
|
||||
if r := []rune(ocr); len(r) > maxOllamaOCRChars {
|
||||
ocr = string(r[:maxOllamaOCRChars])
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("Du bist ein Assistent für ein Dokumentenmanagement-System. ")
|
||||
b.WriteString("Analysiere das folgende Dokument und schlage passende Metadaten vor. ")
|
||||
b.WriteString("Antworte AUSSCHLIESSLICH mit einem JSON-Objekt in genau diesem Schema, ohne weiteren Text:\n")
|
||||
b.WriteString(`{"title": string, "doc_types": [string], "correspondents": [string], "tags": [string]}` + "\n\n")
|
||||
b.WriteString("Regeln:\n")
|
||||
b.WriteString("- Wähle doc_types, correspondents und tags NUR aus den unten aufgelisteten vorhandenen Werten (exakte Schreibweise).\n")
|
||||
b.WriteString("- Wenn nichts passt, gib eine leere Liste zurück.\n")
|
||||
b.WriteString("- title ist ein kurzer, aussagekräftiger Titel für das Dokument.\n\n")
|
||||
|
||||
b.WriteString("Vorhandene document_types: ")
|
||||
b.WriteString(joinEntityNames(docTypes))
|
||||
b.WriteString("\nVorhandene correspondents: ")
|
||||
b.WriteString(joinEntityNames(correspondents))
|
||||
b.WriteString("\nVorhandene tags: ")
|
||||
b.WriteString(joinEntityNames(tags))
|
||||
b.WriteString("\n\n")
|
||||
|
||||
b.WriteString("Aktueller Titel: ")
|
||||
b.WriteString(doc.Title)
|
||||
b.WriteString("\n\nDokumenttext (Auszug):\n")
|
||||
b.WriteString(ocr)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// joinEntityNames renders entity names as a comma-separated list, or "(keine)"
|
||||
// when the tenant has no entities of that kind, so the prompt is never empty.
|
||||
func joinEntityNames(entities []TaxonomyEntity) string {
|
||||
if len(entities) == 0 {
|
||||
return "(keine)"
|
||||
}
|
||||
names := make([]string, 0, len(entities))
|
||||
for _, e := range entities {
|
||||
names = append(names, e.Name)
|
||||
}
|
||||
return strings.Join(names, ", ")
|
||||
}
|
||||
|
||||
// mapNamesToCandidates resolves LLM-returned names to existing taxonomy
|
||||
// entities by exact (case-insensitive) then fuzzy name match, dropping names
|
||||
// that match nothing above ollamaNameMatchFloor, that are already assigned
|
||||
// (excluded), or that duplicate an already-mapped entity. The Score reflects
|
||||
// the name-match confidence. Result is always non-nil, sorted by score desc,
|
||||
// capped at maxSuggestionCandidates.
|
||||
func mapNamesToCandidates(names []string, entities []TaxonomyEntity, excluded func(id int64) bool) []SuggestionCandidate {
|
||||
out := make([]SuggestionCandidate, 0)
|
||||
seen := map[int64]bool{}
|
||||
for _, raw := range names {
|
||||
name := strings.TrimSpace(raw)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
best := TaxonomyEntity{}
|
||||
bestScore := 0.0
|
||||
found := false
|
||||
for _, e := range entities {
|
||||
var score float64
|
||||
if strings.EqualFold(strings.TrimSpace(e.Name), name) {
|
||||
score = 1.0
|
||||
} else {
|
||||
score = matching.FuzzyScore(e.Name, false, name)
|
||||
}
|
||||
if score > bestScore {
|
||||
bestScore = score
|
||||
best = e
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found || bestScore < ollamaNameMatchFloor {
|
||||
continue
|
||||
}
|
||||
if excluded(best.ID) || seen[best.ID] {
|
||||
continue
|
||||
}
|
||||
seen[best.ID] = true
|
||||
out = append(out, SuggestionCandidate{ID: best.ID, Name: best.Name, Score: bestScore})
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Score > out[j].Score })
|
||||
if len(out) > maxSuggestionCandidates {
|
||||
out = out[:maxSuggestionCandidates]
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
-- Dokumentation, siehe README.md. Wird zur Laufzeit idempotent von
|
||||
-- internal/tenantstore, internal/userstore, internal/storage und
|
||||
-- internal/audit initSchema()-Funktionen erzeugt.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tenants (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
slug VARCHAR(100) UNIQUE NOT NULL,
|
||||
domain VARCHAR(255),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_tenants_domain ON tenants (domain) WHERE domain IS NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
username VARCHAR(100) UNIQUE NOT NULL,
|
||||
email VARCHAR(255) UNIQUE NOT NULL,
|
||||
password_hash VARCHAR(255) NOT NULL DEFAULT '',
|
||||
role VARCHAR(20) NOT NULL DEFAULT 'user',
|
||||
active BOOLEAN NOT NULL DEFAULT true,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
last_login_at TIMESTAMPTZ,
|
||||
tenant_id BIGINT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_users_tenant ON users (tenant_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS token_blacklist (
|
||||
jti VARCHAR(255) PRIMARY KEY,
|
||||
expires_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS documents (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
doc_type TEXT,
|
||||
correspondent TEXT,
|
||||
storage_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
ocr_text TEXT,
|
||||
retain_until DATE,
|
||||
source TEXT, -- z.B. 'upload' | 'archivmail_import' (kein Importer implementiert)
|
||||
source_ref TEXT, -- externe Referenz (z.B. archivmail Mail-ID via API-Pull)
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_documents_tenant ON documents(tenant_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS audit_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
event_type VARCHAR(50) NOT NULL,
|
||||
username VARCHAR(255) NOT NULL DEFAULT '',
|
||||
ip_address VARCHAR(45) NOT NULL DEFAULT '',
|
||||
document_id VARCHAR(64) NOT NULL DEFAULT '',
|
||||
success BOOLEAN NOT NULL DEFAULT true,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
tenant_id BIGINT
|
||||
);
|
||||
-- append-only: BEFORE UPDATE OR DELETE trigger raises, see internal/audit/audit.go
|
||||
@@ -0,0 +1,17 @@
|
||||
-- Dokumentation, siehe README.md. Wird zur Laufzeit idempotent von
|
||||
-- internal/storage/reminders.go initReminderSchema() erzeugt.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS reminders (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id),
|
||||
tenant_id BIGINT NOT NULL,
|
||||
user_id BIGINT NOT NULL,
|
||||
due_date TIMESTAMPTZ NOT NULL,
|
||||
note TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'open', -- open|done|dismissed
|
||||
notified_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_reminders_tenant_due ON reminders(tenant_id, due_date) WHERE status = 'open';
|
||||
CREATE INDEX IF NOT EXISTS idx_reminders_document ON reminders(document_id);
|
||||
@@ -0,0 +1,8 @@
|
||||
-- Dokumentation, siehe README.md. Wird zur Laufzeit idempotent von
|
||||
-- internal/storage/documents.go initSchema() erzeugt.
|
||||
|
||||
-- Zusätzlicher Duplikatschutz auf DB-Ebene (neben dem Kollisionscheck auf
|
||||
-- Dateisystemebene im Upload-Handler, siehe internal/api/document_handlers.go
|
||||
-- handleUploadDocument): derselbe Mandant darf denselben Dateiinhalt
|
||||
-- (content_hash) nicht zweimal als aktives Dokument anlegen.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_documents_tenant_hash ON documents(tenant_id, content_hash);
|
||||
@@ -0,0 +1,17 @@
|
||||
-- Dokumentation, siehe README.md. Wird zur Laufzeit idempotent von
|
||||
-- internal/storage/sftp_credentials.go initSFTPCredentialsSchema() erzeugt.
|
||||
|
||||
-- Per-Mandant-Zugangsdaten fuer den eingebetteten SFTP-Server
|
||||
-- (internal/sftpserver). Bewusst getrennt von `users`: ein SFTP-Zugang ist
|
||||
-- ein eigenstaendiges, jederzeit widerrufbares Credential, kein volles
|
||||
-- Login-Konto.
|
||||
CREATE TABLE IF NOT EXISTS sftp_credentials (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
password_hash TEXT NOT NULL, -- bcrypt, analog users.password_hash
|
||||
active BOOLEAN NOT NULL DEFAULT true,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
last_login_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_sftp_credentials_tenant ON sftp_credentials(tenant_id);
|
||||
@@ -0,0 +1,68 @@
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über internal/storage/taxonomy.go
|
||||
-- initTaxonomySchema(), aufgerufen aus (*Store).initSchema().
|
||||
--
|
||||
-- Strukturierte Entitäten (Tags/Dokumenttypen/Korrespondenten) statt der
|
||||
-- bisherigen documents.doc_type/correspondent-Freitextfelder (die bleiben
|
||||
-- unangetastet, Bestandsschutz), plus Barcode-Erkennung fuer automatische
|
||||
-- Zuordnung beim Ingest (siehe internal/matching, internal/barcode).
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
color TEXT,
|
||||
match_algorithm TEXT NOT NULL DEFAULT 'none' CHECK (match_algorithm IN ('none','any','all','exact','regex','fuzzy')),
|
||||
match_pattern TEXT,
|
||||
case_sensitive BOOLEAN NOT NULL DEFAULT false,
|
||||
barcode_value TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_types (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
color TEXT,
|
||||
match_algorithm TEXT NOT NULL DEFAULT 'none' CHECK (match_algorithm IN ('none','any','all','exact','regex','fuzzy')),
|
||||
match_pattern TEXT,
|
||||
case_sensitive BOOLEAN NOT NULL DEFAULT false,
|
||||
barcode_value TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS correspondents (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
color TEXT,
|
||||
match_algorithm TEXT NOT NULL DEFAULT 'none' CHECK (match_algorithm IN ('none','any','all','exact','regex','fuzzy')),
|
||||
match_pattern TEXT,
|
||||
case_sensitive BOOLEAN NOT NULL DEFAULT false,
|
||||
barcode_value TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_tags (
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
tag_id BIGINT NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (document_id, tag_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tags_tenant ON tags(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_types_tenant ON document_types(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_correspondents_tenant ON correspondents(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_tags_tag ON document_tags(tag_id);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_tags_tenant_barcode ON tags(tenant_id, barcode_value) WHERE barcode_value IS NOT NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_document_types_tenant_barcode ON document_types(tenant_id, barcode_value) WHERE barcode_value IS NOT NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_correspondents_tenant_barcode ON correspondents(tenant_id, barcode_value) WHERE barcode_value IS NOT NULL;
|
||||
|
||||
-- documents: neue Spalten, alte doc_type/correspondent-Textspalten bleiben
|
||||
-- unveraendert (Bestandsschutz fuer vorhandene GoBD-Metadaten).
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS doc_type_id BIGINT REFERENCES document_types(id);
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS correspondent_id BIGINT REFERENCES correspondents(id);
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS barcode_values JSONB;
|
||||
@@ -0,0 +1,44 @@
|
||||
-- PROJ: custom-fields
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über internal/storage/custom_fields.go
|
||||
-- initCustomFieldsSchema(), aufgerufen aus (*Store).initSchema().
|
||||
--
|
||||
-- Benutzerdefinierte Felder (Custom Fields): tenant-skopierte Feld-
|
||||
-- Definitionen, Zuordnung pro Dokumenttyp (required/visible/sort_order) und
|
||||
-- die eigentlichen Werte pro Dokument. Regeln: enum-Wert liegt in value_text,
|
||||
-- monetary in value_number (NUMERIC(14,2) semantisch).
|
||||
|
||||
CREATE TABLE IF NOT EXISTS custom_field_defs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
label TEXT NOT NULL,
|
||||
field_type TEXT NOT NULL CHECK (field_type IN ('text','number','date','boolean','enum','monetary')),
|
||||
enum_options JSONB,
|
||||
currency TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_type_fields (
|
||||
doc_type_id BIGINT NOT NULL REFERENCES document_types(id) ON DELETE CASCADE,
|
||||
field_id BIGINT NOT NULL REFERENCES custom_field_defs(id) ON DELETE CASCADE,
|
||||
required BOOLEAN NOT NULL DEFAULT false,
|
||||
visible BOOLEAN NOT NULL DEFAULT true,
|
||||
sort_order INT NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (doc_type_id, field_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_field_values (
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
field_id BIGINT NOT NULL REFERENCES custom_field_defs(id) ON DELETE CASCADE,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
value_text TEXT,
|
||||
value_number NUMERIC,
|
||||
value_date DATE,
|
||||
value_bool BOOLEAN,
|
||||
PRIMARY KEY (document_id, field_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_dfv_tenant_field ON document_field_values(tenant_id, field_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_dfv_field_text ON document_field_values(field_id, value_text);
|
||||
CREATE INDEX IF NOT EXISTS idx_dfv_field_number ON document_field_values(field_id, value_number);
|
||||
@@ -0,0 +1,36 @@
|
||||
-- PROJ: trash-staged-deletion
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über internal/storage/trash.go
|
||||
-- initTrashSchema(), aufgerufen aus (*Store).initSchema().
|
||||
--
|
||||
-- Papierkorb + gestaffeltes Löschkonzept (GoBD):
|
||||
-- * documents.deleted_at/deleted_by = Soft-Delete (Papierkorb). Die WORM-Datei
|
||||
-- bleibt physisch unangetastet (chmod 0440) bis ein Löschantrag den Status
|
||||
-- 'executed' erreicht.
|
||||
-- * document_delete_requests = Vier-/Zwei-Augen-Workflow für finales Löschen:
|
||||
-- User A stellt Antrag ('pending'), ein anderer domain_admin (User B)
|
||||
-- bestätigt ('confirmed'->'executed'). Retention (retain_until) wird bei
|
||||
-- Antrag UND Bestätigung geprüft; ein blockierter Versuch wird als
|
||||
-- 'blocked_retention' protokolliert (Nachvollziehbarkeit).
|
||||
-- * Finales Löschen: physische Datei via os.Remove entfernt, DB-Row als
|
||||
-- Tombstone behalten (storage_path/ocr_text geleert, content_hash bleibt).
|
||||
-- * Das Vier-Augen-Prinzip (confirmed_by != requested_by) wird im Go-Store
|
||||
-- erzwungen, NICHT per CHECK-Constraint (confirmed_by wird erst später
|
||||
-- gesetzt).
|
||||
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS deleted_by BIGINT;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_delete_requests (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
requested_by BIGINT NOT NULL,
|
||||
requested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
confirmed_by BIGINT,
|
||||
confirmed_at TIMESTAMPTZ,
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending','confirmed','executed','cancelled','blocked_retention')),
|
||||
UNIQUE(document_id, status)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ddr_tenant_status ON document_delete_requests(tenant_id, status);
|
||||
@@ -0,0 +1,78 @@
|
||||
-- PROJ: permission-model-group-acl
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über internal/storage/permissions.go
|
||||
-- initPermissionsSchema(), aufgerufen aus (*Store).New().
|
||||
--
|
||||
-- Berechtigungsmodell (gruppenaufgelöste, geschichtete Dokument-ACL):
|
||||
-- Zugriff wird NIE direkt pro User vergeben, sondern über permission_groups.
|
||||
-- Auflösung über drei Ebenen, spezifischer schlägt allgemeiner:
|
||||
-- 1. document_grants (pro Dokument, 'deny' entfernt eine Gruppe komplett)
|
||||
-- 2. tag_grants (über die Tags des Dokuments)
|
||||
-- 3. document_type_grants (über documents.doc_type_id)
|
||||
-- Ergebnis wird von RecomputeVisibility() nach document_visibility
|
||||
-- materialisiert (DELETE+INSERT je Dokument in einer Transaktion).
|
||||
--
|
||||
-- Rollen bleiben Außengrenze: superadmin sieht alles tenant-übergreifend,
|
||||
-- domain_admin sieht per Default alles im eigenen Tenant (bypasst ACL), nur
|
||||
-- Rolle 'user' wird in ListDocuments gegen document_visibility gefiltert.
|
||||
--
|
||||
-- RecomputeVisibility MUSS neu laufen, wenn sich Grants, die Tags eines
|
||||
-- Dokuments (AttachTag/DetachTag) oder der Dokumenttyp (SetDocumentDocType)
|
||||
-- ändern — diese Aufrufe sind im Go-Code angehängt.
|
||||
|
||||
-- Hinweis: tenant_id / user_id / granted_by tragen bewusst KEINE FK auf
|
||||
-- tenants(id) bzw. users(id) — konsistent mit documents/taxonomy (plain BIGINT
|
||||
-- tenant_id) und weil tenants/users von anderen Stores NACH storage.New()
|
||||
-- angelegt werden (Reihenfolge in cmd/archivdms/main.go). Tenant-Ownership wird
|
||||
-- applikationsseitig geprüft (permissions.go).
|
||||
CREATE TABLE IF NOT EXISTS permission_groups (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (tenant_id, name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS permission_group_members (
|
||||
group_id BIGINT NOT NULL REFERENCES permission_groups(id) ON DELETE CASCADE,
|
||||
user_id BIGINT NOT NULL,
|
||||
PRIMARY KEY (group_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_type_grants (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
doc_type_id BIGINT NOT NULL REFERENCES document_types(id) ON DELETE CASCADE,
|
||||
group_id BIGINT NOT NULL REFERENCES permission_groups(id) ON DELETE CASCADE,
|
||||
access TEXT NOT NULL DEFAULT 'read' CHECK (access IN ('read','write')),
|
||||
UNIQUE (doc_type_id, group_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tag_grants (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
tag_id BIGINT NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
|
||||
group_id BIGINT NOT NULL REFERENCES permission_groups(id) ON DELETE CASCADE,
|
||||
access TEXT NOT NULL DEFAULT 'read' CHECK (access IN ('read','write')),
|
||||
UNIQUE (tag_id, group_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_grants (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
group_id BIGINT NOT NULL REFERENCES permission_groups(id) ON DELETE CASCADE,
|
||||
access TEXT NOT NULL CHECK (access IN ('read','write','deny')),
|
||||
granted_by BIGINT NOT NULL,
|
||||
granted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (document_id, group_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_visibility (
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
group_id BIGINT NOT NULL REFERENCES permission_groups(id) ON DELETE CASCADE,
|
||||
access TEXT NOT NULL CHECK (access IN ('read','write')),
|
||||
PRIMARY KEY (document_id, group_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_doc_visibility_group ON document_visibility(group_id, document_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_pgm_user ON permission_group_members(user_id, group_id);
|
||||
@@ -0,0 +1,53 @@
|
||||
-- PROJ: external-share-links
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über internal/storage/shares.go
|
||||
-- initSharesSchema(), aufgerufen aus (*Store).New().
|
||||
--
|
||||
-- Externe Share-Links: tenant-scoped, ablaufender, optional passwortgeschützter
|
||||
-- öffentlicher Link auf genau EIN Dokument.
|
||||
-- * Token: 32 Byte crypto/rand, base64url; wird beim Erzeugen genau EINMAL
|
||||
-- zurückgegeben. Persistiert wird NUR der SHA-256-Hash (token_hash). Der
|
||||
-- öffentliche Abruf sucht immer über token_hash, nie über id.
|
||||
-- * expires_at ist Pflicht (kein unbegrenzter Share). password_hash optional
|
||||
-- (bcrypt, Cost 12).
|
||||
-- * Kein Hard-Delete: Revoke setzt nur revoked_at/revoked_by.
|
||||
-- * Prüfreihenfolge beim öffentlichen Abruf: revoked -> expired ->
|
||||
-- max_accesses erreicht -> Passwort -> ausliefern (access_count++ atomar
|
||||
-- mit WHERE-Guard gegen Race). Datei wird serverseitig aus dem WORM-Store
|
||||
-- gestreamt; storage_path/content_hash NIE im Response.
|
||||
-- * Jeder Zugriffsversuch (Erfolg/Fehlschlag) landet in
|
||||
-- document_share_accesses mit passendem result-Wert; die öffentlichen
|
||||
-- Endpunkte sind zusätzlich per-IP rate-limited (Token-Bucket in-memory).
|
||||
--
|
||||
-- Hinweis: tenant_id / created_by / revoked_by tragen bewusst KEINE FK auf
|
||||
-- tenants(id) bzw. users(id) — konsistent mit documents/taxonomy/permissions
|
||||
-- (plain BIGINT) und weil tenants/users von anderen Stores NACH storage.New()
|
||||
-- angelegt werden (Reihenfolge in cmd/archivdms/main.go). document_id behält
|
||||
-- seine FK, da documents die eigene Tabelle dieses Stores ist.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_shares (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
token_hash TEXT NOT NULL UNIQUE,
|
||||
created_by BIGINT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
max_accesses INT,
|
||||
access_count INT NOT NULL DEFAULT 0,
|
||||
password_hash TEXT,
|
||||
revoked_at TIMESTAMPTZ,
|
||||
revoked_by BIGINT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_shares_document ON document_shares(document_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_shares_tenant ON document_shares(tenant_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_share_accesses (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
share_id BIGINT NOT NULL REFERENCES document_shares(id) ON DELETE CASCADE,
|
||||
accessed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
ip_address INET,
|
||||
user_agent TEXT,
|
||||
result TEXT NOT NULL CHECK (result IN ('success','expired','revoked','max_reached','bad_password','rate_limited'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_share_accesses_share ON document_share_accesses(share_id, accessed_at DESC);
|
||||
@@ -0,0 +1,42 @@
|
||||
-- PROJ: manticore-search-index-phase1
|
||||
-- Doku-only. KEINE PostgreSQL-Schema-Änderung in dieser Phase — Postgres
|
||||
-- (documents + Taxonomie + document_visibility) bleibt Source of Truth und
|
||||
-- unverändert. Diese Datei dokumentiert nur den externen Volltext-Index.
|
||||
--
|
||||
-- Phase 1 der geplanten Manticore-Search-Integration (Hybrid BM25+Vektor
|
||||
-- kommt später): NUR Schema + Sync-Layer, KEIN Such-Endpunkt.
|
||||
--
|
||||
-- Der Index läuft in Manticore Search (MySQL-Protokoll, Default Port 9306),
|
||||
-- angesprochen über internal/index (github.com/go-sql-driver/mysql, CGO-frei).
|
||||
-- Pro Mandant existiert eine RT-Tabelle documents_tenant_<tenant_id>, die
|
||||
-- idempotent von ensureTable() angelegt wird:
|
||||
--
|
||||
-- CREATE TABLE documents_tenant_N (
|
||||
-- doc_id string,
|
||||
-- title text,
|
||||
-- doc_type text,
|
||||
-- correspondent text,
|
||||
-- ocr_text text,
|
||||
-- tags text,
|
||||
-- tag_ids multi,
|
||||
-- doc_type_id bigint,
|
||||
-- correspondent_id bigint,
|
||||
-- acl_group_ids multi,
|
||||
-- retain_until_ts bigint,
|
||||
-- created_ts bigint,
|
||||
-- updated_ts bigint,
|
||||
-- deleted uint
|
||||
-- ) type='rt' morphology='lemmatize_de_all,stem_en'
|
||||
--
|
||||
-- Aktivierung nur wenn index.manticore_dsn in der config.yml gesetzt ist —
|
||||
-- sonst ist der Indexer nil und alle Sync-Aufrufe sind No-ops.
|
||||
--
|
||||
-- Sync-Punkte (alle best-effort, Fehler werden nur geloggt, blockieren nie den
|
||||
-- Haupt-Request — Postgres bleibt maßgeblich):
|
||||
-- * Dokument-Upload/Create -> IndexSync
|
||||
-- * RecomputeVisibility (ACL/Tags/DocType) -> IndexSync
|
||||
-- * SetDocumentCorrespondent -> IndexSync
|
||||
-- * Custom-Field-Werte setzen -> IndexSync
|
||||
-- * SoftDeleteDocument (Papierkorb) -> Delete (nicht mehr auffindbar)
|
||||
-- * RestoreDocument -> IndexSync (wieder auffindbar)
|
||||
-- * ConfirmDeleteRequest (final/executed) -> Delete (GoBD: endgültig weg)
|
||||
@@ -0,0 +1,52 @@
|
||||
-- PROJ: classification-templates
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über
|
||||
-- internal/storage/classification_templates.go
|
||||
-- initClassificationTemplatesSchema(), aufgerufen aus (*Store).initSchema()
|
||||
-- NACH initTaxonomySchema/initCustomFieldsSchema (FK auf document_types und
|
||||
-- custom_field_defs).
|
||||
--
|
||||
-- Klassifizierungsvorlagen (classification templates): tenant-skopierte,
|
||||
-- benannte Bündel aus Dokumenttyp, Tags, Custom-Field-Defaultwerten und einer
|
||||
-- Aufbewahrungsdauer, die in einem Schritt auf ein Dokument angewendet werden
|
||||
-- können. Bewusst KEINE persistente Kopplung template<->document (keine
|
||||
-- template_id-Spalte auf documents): eine spätere Vorlagen-Änderung darf
|
||||
-- Bestandsdokumente niemals rückwirkend verändern (GoBD-Nachvollziehbarkeit).
|
||||
-- Die Anwendung wird ausschliesslich über einen Audit-Log-Eintrag festgehalten.
|
||||
--
|
||||
-- Retention-Regel (absolut, ohne Ausnahme): eine Vorlage kann retain_until nur
|
||||
-- verlängern, niemals verkürzen — auch nicht mit overwrite=true. Ein
|
||||
-- abgelehnter Verkürzungsversuch wird als Audit-Eintrag protokolliert.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS classification_templates (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
doc_type_id BIGINT REFERENCES document_types(id) ON DELETE SET NULL,
|
||||
retain_years INT,
|
||||
active BOOLEAN NOT NULL DEFAULT true,
|
||||
created_by BIGINT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, name)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_classification_templates_tenant ON classification_templates(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_classification_templates_doc_type ON classification_templates(doc_type_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS classification_template_tags (
|
||||
template_id BIGINT NOT NULL REFERENCES classification_templates(id) ON DELETE CASCADE,
|
||||
tag_id BIGINT NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (template_id, tag_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS classification_template_field_defaults (
|
||||
template_id BIGINT NOT NULL REFERENCES classification_templates(id) ON DELETE CASCADE,
|
||||
field_id BIGINT NOT NULL REFERENCES custom_field_defs(id) ON DELETE CASCADE,
|
||||
value_text TEXT,
|
||||
value_number NUMERIC,
|
||||
value_date DATE,
|
||||
value_bool BOOLEAN,
|
||||
overwrite BOOLEAN NOT NULL DEFAULT false,
|
||||
PRIMARY KEY (template_id, field_id)
|
||||
);
|
||||
@@ -0,0 +1,33 @@
|
||||
-- PROJ: heuristische Metadaten-Vorschläge
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über
|
||||
-- internal/storage/metadata_suggestions.go initMetadataSuggestionsSchema(),
|
||||
-- aufgerufen aus (*Store).initSchema() NACH dem documents-/taxonomy-Schema
|
||||
-- (die Vorschläge scoren Taxonomie-Entitäten gegen ein Dokument).
|
||||
--
|
||||
-- Metadaten-Vorschläge: regelbasiert (KEIN LLM). Pro Vorschlags-Lauf wird ein
|
||||
-- Dokument gegen alle Taxonomie-Entitäten (Tags/Dokumenttypen/Korrespondenten)
|
||||
-- fuzzy-gescored; Near-Misses oberhalb einer Schwelle (suggestionFloor), die
|
||||
-- noch NICHT zugewiesen sind, werden als nicht-bindende Kandidaten vorgeschlagen.
|
||||
-- Sieht der aktuelle Titel noch auto-generiert aus, wird zusätzlich ein
|
||||
-- neu abgeleiteter Titel vorgeschlagen.
|
||||
--
|
||||
-- Diese Tabelle ist NUR ein Log/Cache dessen, was der heuristische Provider
|
||||
-- vorgeschlagen hat. Das Anwenden eines akzeptierten Feldes läuft über die
|
||||
-- normalen Edit-Endpunkte (PATCH title, tag-attach, ...), NIEMALS über diese
|
||||
-- Zeile. status wird beim Review ('reviewed') gesetzt, unabhängig davon welche
|
||||
-- Felder der Nutzer übernommen hat.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS metadata_suggestions (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
document_id BIGINT NOT NULL,
|
||||
provider TEXT NOT NULL DEFAULT 'heuristic',
|
||||
requested_by BIGINT,
|
||||
requested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
suggestion JSONB NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','reviewed')),
|
||||
reviewed_at TIMESTAMPTZ,
|
||||
reviewed_by BIGINT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_metadata_suggestions_document ON metadata_suggestions(document_id);
|
||||
@@ -0,0 +1,57 @@
|
||||
-- PROJ: workflows / Consumption-Regeln
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über internal/storage/workflows.go
|
||||
-- initWorkflowsSchema(), aufgerufen aus (*Store).initSchema() NACH
|
||||
-- initClassificationTemplatesSchema (eine Workflow-Action kann eine
|
||||
-- Klassifizierungsvorlage referenzieren: apply_classification_template).
|
||||
--
|
||||
-- Workflows (Consumption-Regeln): tenant-skopierte Automatisierungsregeln, die
|
||||
-- an einem Trigger-Punkt (MVP: on_upload) ausgewertet werden. Ein passender
|
||||
-- Workflow führt seine geordneten Actions auf dem Dokument aus. Die
|
||||
-- Bedingung ist ein JSONB-Condition-Tree (bool. Gruppen and/or + Leaf-Knoten
|
||||
-- mit Feld/Algorithmus/Pattern, MVP-Verschachtelungstiefe 2). Jede Auswertung
|
||||
-- wird dokument-skopiert in workflow_runs protokolliert (GoBD-Reproduzierbarkeit
|
||||
-- parallel zum globalen internal/audit-Log).
|
||||
--
|
||||
-- Best-effort-Ausführung: eine fehlgeschlagene Action bricht den Upload niemals
|
||||
-- ab — Fehler werden in workflow_runs.error festgehalten und die Schleife läuft
|
||||
-- weiter (spiegelt die "never fail the upload"-Philosophie der OCR-/
|
||||
-- Auto-Assign-Schritte in storeUploadedFile).
|
||||
|
||||
CREATE TABLE IF NOT EXISTS workflows (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
enabled BOOLEAN NOT NULL DEFAULT true,
|
||||
trigger_type TEXT NOT NULL CHECK (trigger_type IN ('on_upload')),
|
||||
condition_tree JSONB NOT NULL,
|
||||
priority INT NOT NULL DEFAULT 100,
|
||||
created_by BIGINT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, name)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_workflows_tenant ON workflows(tenant_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS workflow_actions (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
workflow_id BIGINT NOT NULL REFERENCES workflows(id) ON DELETE CASCADE,
|
||||
step_order INT NOT NULL,
|
||||
action_type TEXT NOT NULL CHECK (action_type IN
|
||||
('add_tag','set_doc_type','set_correspondent','apply_classification_template','set_custom_field')),
|
||||
action_config JSONB NOT NULL DEFAULT '{}'
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_workflow_actions_workflow ON workflow_actions(workflow_id, step_order);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS workflow_runs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
workflow_id BIGINT NOT NULL REFERENCES workflows(id) ON DELETE CASCADE,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
document_id BIGINT,
|
||||
triggered_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
matched BOOLEAN NOT NULL,
|
||||
actions_applied JSONB,
|
||||
error TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_workflow_runs_workflow ON workflow_runs(workflow_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_workflow_runs_document ON workflow_runs(document_id);
|
||||
@@ -0,0 +1,21 @@
|
||||
-- PROJ: Freitext-Notizen pro Dokument (Paperless-ngx inspiriert)
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über
|
||||
-- internal/storage/document_notes.go initDocumentNotesSchema(), aufgerufen aus
|
||||
-- (*Store).initSchema() NACH dem documents-Schema (FK auf documents(id)).
|
||||
--
|
||||
-- Abgrenzung zu Custom-Fields: Custom-Fields sind strukturierte Metadaten;
|
||||
-- eine Notiz ist reiner Freitext-Kommentar mit Autor + Zeitstempel. Notizen
|
||||
-- sind KEINE GoBD-Belege, deshalb hartes DELETE (kein Soft-Delete). Create und
|
||||
-- Delete werden dennoch im Audit-Log protokolliert (Nachvollziehbarkeit).
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_notes (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id),
|
||||
tenant_id BIGINT NOT NULL,
|
||||
author_id BIGINT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_notes_document ON document_notes (document_id, tenant_id);
|
||||
@@ -0,0 +1,25 @@
|
||||
-- PROJ: Gespeicherte Suchansichten (SavedViews, Paperless-ngx inspiriert)
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über
|
||||
-- internal/storage/saved_views.go initSavedViewsSchema(), aufgerufen aus
|
||||
-- (*Store).initSchema() NACH initDocumentNotesSchema.
|
||||
--
|
||||
-- Nutzer speichern ihre aktuelle Such-/Filter-Query als benannte,
|
||||
-- wiederverwendbare Ansicht. filters ist die serialisierte index.SearchQuery
|
||||
-- (JSONB, 1:1 wieder einlesbar). Eine Ansicht ist privat für ihren Ersteller,
|
||||
-- außer is_shared=true -> tenant-weit sichtbar, aber weiterhin nur vom
|
||||
-- Ersteller änderbar/löschbar (WHERE id + tenant_id + user_id). Create/Update/
|
||||
-- Delete werden im Audit-Log protokolliert (Nachvollziehbarkeit).
|
||||
|
||||
CREATE TABLE IF NOT EXISTS saved_views (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
user_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
filters JSONB NOT NULL,
|
||||
is_shared BOOLEAN NOT NULL DEFAULT false,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_saved_views_tenant_user ON saved_views (tenant_id, user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_saved_views_tenant_shared ON saved_views (tenant_id, is_shared) WHERE is_shared;
|
||||
@@ -0,0 +1,20 @@
|
||||
-- PROJ: Pro-Mandant konfigurierbares Datumsformat für Platzhalter-Titel
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über
|
||||
-- internal/tenantstore/store.go (*Store).initSchema().
|
||||
--
|
||||
-- Beim Upload wird, wenn kein Titel angegeben ist und OCR keine sinnvolle
|
||||
-- Überschrift liefert, ein Platzhalter-Titel "Scan <Datum>" erzeugt
|
||||
-- (titleFromOCRText in internal/api/document_handlers.go). Das Datumsformat
|
||||
-- war bisher hart auf DD.MM.YYYY HH:mm codiert. Diese Spalte macht es pro
|
||||
-- Mandant (nicht global, nicht pro Nutzer) konfigurierbar.
|
||||
--
|
||||
-- Gespeichert wird EINER von wenigen bekannten Format-Schlüsseln (kein roher
|
||||
-- Go-Layout-String vom Nutzer), das Backend mappt den Schlüssel auf das
|
||||
-- Go-Layout und validiert den Wert (400 bei Unbekannt). Erlaubte Schlüssel:
|
||||
-- 'DD.MM.YYYY HH:mm' -> 02.01.2006 15:04 (deutsch, Default)
|
||||
-- 'YYYY-MM-DD HH:mm' -> 2006-01-02 15:04 (ISO)
|
||||
-- 'MM/DD/YYYY hh:mm AM/PM' -> 01/02/2006 03:04 PM (US)
|
||||
|
||||
ALTER TABLE tenants
|
||||
ADD COLUMN IF NOT EXISTS scan_title_date_format TEXT NOT NULL DEFAULT 'DD.MM.YYYY HH:mm';
|
||||
@@ -0,0 +1,19 @@
|
||||
-- PROJ: Pro-Mandant konfigurierbares Präfix-Wort für Platzhalter-Titel
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über
|
||||
-- internal/tenantstore/store.go (*Store).initSchema().
|
||||
--
|
||||
-- Beim Upload wird, wenn kein Titel angegeben ist und OCR keine sinnvolle
|
||||
-- Überschrift liefert, ein Platzhalter-Titel "<Präfix> <Datum>" erzeugt
|
||||
-- (titleFromOCRText in internal/api/document_handlers.go). Das Präfix-Wort
|
||||
-- war bisher hart auf "Scan" codiert. Diese Spalte macht es pro Mandant
|
||||
-- (nicht global, nicht pro Nutzer) konfigurierbar, z.B. "Beleg", "Import",
|
||||
-- "Eingang".
|
||||
--
|
||||
-- Validierung im Backend (internal/tenantstore/store.go UpdateScanTitlePrefix):
|
||||
-- nicht leer, maximal 40 Zeichen (400 bei Verstoß). Ergänzt Migration 015
|
||||
-- (scan_title_date_format), beide Settings sind unabhängig änderbar über
|
||||
-- GET/PUT /api/tenant-settings.
|
||||
|
||||
ALTER TABLE tenants
|
||||
ADD COLUMN IF NOT EXISTS scan_title_prefix TEXT NOT NULL DEFAULT 'Scan';
|
||||
@@ -0,0 +1,26 @@
|
||||
-- PROJ: Pro-Mandant konfigurierbare Anbindung an einen EXTERNEN Ollama-Server
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über
|
||||
-- internal/storage/documents.go (*Store).initSchema() -> initOllamaConfigSchema
|
||||
-- (internal/storage/ollama_config.go). Source of Truth bleibt der Go-Code.
|
||||
--
|
||||
-- Ollama läuft NICHT lokal auf dem archivdms-Host, sondern als bereits
|
||||
-- laufender externer Dienst; IP/Port kommt vom Mandanten-Admin. Diese Tabelle
|
||||
-- speichert die Verbindung PRO Mandant (analog ldap_configs), nicht global.
|
||||
-- Sie schaltet den optionalen 'ollama'-Provider der Metadaten-Vorschläge frei
|
||||
-- (metadata_suggestions.provider), neben dem bestehenden 'heuristic'-Provider.
|
||||
--
|
||||
-- Validierung im Backend (UpsertOllamaConfig): bei enabled=true muss base_url
|
||||
-- (http:// oder https:// Präfix) und model gesetzt sein, timeout_seconds in
|
||||
-- [5,120]. base_url ist eine interne Netzwerk-URL, kein Secret, und wird von
|
||||
-- GET/PUT /api/ollama-config normal zurückgegeben (kein Masking wie bei der
|
||||
-- LDAP-Bind-Passwort-Spalte).
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tenant_ollama_config (
|
||||
tenant_id BIGINT PRIMARY KEY REFERENCES tenants(id),
|
||||
enabled BOOLEAN NOT NULL DEFAULT false,
|
||||
base_url TEXT NOT NULL DEFAULT '',
|
||||
model TEXT NOT NULL DEFAULT '',
|
||||
timeout_seconds INT NOT NULL DEFAULT 30,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -0,0 +1,31 @@
|
||||
-- PROJ: Digitale Akte (digitaler Aktenordner) — Gruppierung von Dokumenten.
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über
|
||||
-- internal/storage/documents.go (*Store).initSchema() -> initAktenSchema
|
||||
-- (internal/storage/akten.go). Source of Truth bleibt der Go-Code.
|
||||
--
|
||||
-- Strikt 1:n zu Dokumenten via documents.akte_id (kein Join-Table): ein
|
||||
-- Dokument gehört zu maximal einer Akte. ON DELETE SET NULL ist die
|
||||
-- GoBD-Absicherung — eine Akte löschen kann strukturell nie Dokumente löschen,
|
||||
-- sondern entkoppelt sie nur. Keine eigene ACL: die Sichtbarkeit einer Akte
|
||||
-- erbt von den enthaltenen Dokumenten (EXISTS gegen document_visibility,
|
||||
-- gleiches Pattern wie ListDocuments). Siehe project_akte_konzept_plan.md.
|
||||
--
|
||||
-- Reihenfolge: akten-Tabelle muss VOR der ALTER TABLE auf documents existieren,
|
||||
-- da documents.akte_id auf akten(id) verweist. correspondent_id verweist auf
|
||||
-- correspondents(id) (initTaxonomySchema läuft davor).
|
||||
|
||||
CREATE TABLE IF NOT EXISTS akten (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
titel TEXT NOT NULL,
|
||||
beschreibung TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'offen' CHECK (status IN ('offen','geschlossen')),
|
||||
correspondent_id BIGINT REFERENCES correspondents(id),
|
||||
created_by BIGINT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
closed_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_akten_tenant ON akten (tenant_id);
|
||||
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS akte_id BIGINT REFERENCES akten(id) ON DELETE SET NULL;
|
||||
@@ -0,0 +1,23 @@
|
||||
-- PROJ: Beleg-/Dokumentdatum (document_date) aus dem OCR-Text
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über
|
||||
-- internal/storage/documents.go (*Store).initSchema().
|
||||
--
|
||||
-- Bisher wurde der WORM-Ablagepfad store/<tenant>/<yyyy>/<mm>/<hash>.<ext>
|
||||
-- aus dem Scan-/Upload-Zeitpunkt (time.Now()) gebildet. Neu wird beim Upload
|
||||
-- versucht, das echte Belegdatum (Rechnungs-/Dokumentdatum) aus dem OCR-Text
|
||||
-- zu erkennen (extractDocumentDate in internal/api/date_extraction.go,
|
||||
-- Regex-basiert: DD.MM.YYYY, DD.MM.YY, YYYY-MM-DD). Wird ein plausibles Datum
|
||||
-- gefunden, bestimmt dessen Jahr/Monat den Ablagepfad; sonst Fallback auf den
|
||||
-- Scan-Zeitpunkt wie bisher.
|
||||
--
|
||||
-- created_at bleibt unverändert der unveränderliche Scan-/Upload-Zeitstempel
|
||||
-- (GoBD-Nachvollziehbarkeit). document_date ist ein SEPARATES, nullbares
|
||||
-- Metadatenfeld: NULL bei Altbeständen und wenn kein Datum erkannt wurde.
|
||||
--
|
||||
-- Reprocess (POST /api/documents/{id}/reprocess) aktualisiert bei erneuter
|
||||
-- OCR-Verarbeitung nur dieses Feld — die bereits WORM-gesperrte Datei und ihr
|
||||
-- Ablagepfad werden NIEMALS nachträglich verschoben.
|
||||
|
||||
ALTER TABLE documents
|
||||
ADD COLUMN IF NOT EXISTS document_date DATE;
|
||||
@@ -0,0 +1,55 @@
|
||||
-- PROJ: ML-Retraining-Klassifizierung Phase 1 (Naive-Bayes)
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über
|
||||
-- internal/storage/ml_classifier.go (*Store).initMLClassifierSchema(),
|
||||
-- eingehängt in internal/storage/documents.go (*Store).initSchema() NACH
|
||||
-- initTaxonomySchema (setzt document_types/correspondents/tags/document_tags/
|
||||
-- documents voraus).
|
||||
--
|
||||
-- Ergänzt die bestehende Regel-Engine (Taxonomie-Matching) um ein optionales,
|
||||
-- pro Mandant trainiertes Naive-Bayes-Modell: aus bereits klassifizierten
|
||||
-- Dokumenten werden Token-Häufigkeiten je Klasse (document_type/correspondent/
|
||||
-- tag) gelernt (ml_classifier_tokens/ml_classifier_classes) und je Trainingslauf
|
||||
-- protokolliert (ml_classifier_runs).
|
||||
--
|
||||
-- Provenienz-Spalten (assigned_via auf document_tags, doc_type_assigned_via/
|
||||
-- correspondent_assigned_via auf documents) unterscheiden 'manual' (Nutzer
|
||||
-- hat gesetzt), 'rule' (Regel-Engine) und 'ml_accepted' (vom Klassifizierer
|
||||
-- vorgeschlagen und vom Nutzer akzeptiert). DEFAULT 'manual' bewusst konservativ
|
||||
-- gewählt: bestehende Zeilen werden NICHT rückwirkend als 'rule' fehlklassifiziert
|
||||
-- und fließen dadurch weiterhin normal als Trainingsdaten in den Klassifizierer ein.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ml_classifier_tokens (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('document_types','correspondents','tags')),
|
||||
entity_id BIGINT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
count BIGINT NOT NULL DEFAULT 0,
|
||||
UNIQUE (tenant_id, kind, entity_id, token)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ml_tokens_lookup ON ml_classifier_tokens(tenant_id, kind, token);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ml_classifier_classes (
|
||||
tenant_id BIGINT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
entity_id BIGINT NOT NULL,
|
||||
doc_count BIGINT NOT NULL DEFAULT 0,
|
||||
total_tokens BIGINT NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (tenant_id, kind, entity_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ml_classifier_runs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
completed_at TIMESTAMPTZ,
|
||||
doc_count BIGINT NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'running' CHECK (status IN ('running','completed','failed','skipped_insufficient_data')),
|
||||
error TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ml_classifier_runs_tenant ON ml_classifier_runs(tenant_id);
|
||||
|
||||
ALTER TABLE document_tags ADD COLUMN IF NOT EXISTS assigned_via TEXT NOT NULL DEFAULT 'manual' CHECK (assigned_via IN ('manual','rule','ml_accepted'));
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS doc_type_assigned_via TEXT NOT NULL DEFAULT 'manual' CHECK (doc_type_assigned_via IN ('manual','rule','ml_accepted'));
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS correspondent_assigned_via TEXT NOT NULL DEFAULT 'manual' CHECK (correspondent_assigned_via IN ('manual','rule','ml_accepted'));
|
||||
@@ -0,0 +1,31 @@
|
||||
-- PROJ: Titel-Vorlage für Klassifizierungsvorlagen + globaler Tenant-Default
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über die Go-initSchema-Aufrufe:
|
||||
-- * classification_templates.title_template ->
|
||||
-- internal/storage/classification_templates.go (*Store).initClassificationTemplatesSchema()
|
||||
-- * tenants.default_title_template ->
|
||||
-- internal/tenantstore/store.go (*Store).initSchema()
|
||||
--
|
||||
-- Beim Anwenden einer Klassifizierungsvorlage (POST /api/documents/{id}/apply-template
|
||||
-- ODER Workflow-Aktion apply_classification_template, gemeinsamer Code-Pfad in
|
||||
-- internal/storage/classification_templates_apply.go ApplyTemplate) wird der
|
||||
-- Dokumenttitel aus einer Go-text/template-Vorlage abgeleitet:
|
||||
-- 1. title_template der Vorlage (NULL = keine),
|
||||
-- 2. sonst tenant-weites default_title_template (NULL = keins),
|
||||
-- 3. sonst bleibt der bestehende Titel unangetastet.
|
||||
-- Gesetzt wird NUR wenn documents.title_manually_set = false; title_manually_set
|
||||
-- bleibt danach false (erneute Anwendung nach späterer Korrektur möglich). Bei
|
||||
-- leerem/fehlerhaftem Render-Ergebnis Fallback auf den bestehenden Titel, nie
|
||||
-- leerer String.
|
||||
--
|
||||
-- Platzhalter (Struct-Felder): {{.Correspondent}} {{.DocumentType}}
|
||||
-- {{.Belegdatum}} {{.UploadDate}} {{.Tags}} {{.OCRTitle}}
|
||||
-- Custom-Func dateFormat "02.01.2006" .Belegdatum (Go-Referenzdatum, leer bei
|
||||
-- unbekanntem Datum). Validierung: internal/storage/classification_templates_title.go
|
||||
-- ValidateTitleTemplate (Parse ohne Execute). Tenant-Default max. 500 Zeichen.
|
||||
|
||||
ALTER TABLE classification_templates
|
||||
ADD COLUMN IF NOT EXISTS title_template TEXT;
|
||||
|
||||
ALTER TABLE tenants
|
||||
ADD COLUMN IF NOT EXISTS default_title_template TEXT;
|
||||
@@ -0,0 +1,53 @@
|
||||
-- PROJ: retention-rules-engine
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über internal/storage/retention_rules.go
|
||||
-- initRetentionRulesSchema(), aufgerufen aus (*Store).initSchema().
|
||||
--
|
||||
-- GoBD-Aufbewahrungsregeln (retention rules / "Disposition Schedules",
|
||||
-- Namens-/Modellreferenz Alfresco, NICHT dessen Architektur):
|
||||
-- * Eine Regel definiert pro Dokumenttyp (oder tenant-weit als Default mit
|
||||
-- doc_type_id IS NULL) WIE LANGE ein Dokument aufbewahrt werden muss und
|
||||
-- ab WELCHEM Stichtag (trigger_type) die Frist zählt.
|
||||
-- * Der Batch-Job ApplyRetentionRules (CLI: `archivdms retention apply`)
|
||||
-- berechnet retain_until und SETZT es auf documents — er löscht NIEMALS
|
||||
-- und verkürzt eine bereits gesetzte Sperre NIE (GoBD: WORM nur
|
||||
-- verlängerbar). Die eigentliche Vernichtung läuft weiter über den
|
||||
-- Papierkorb + Vier-Augen-Workflow (007_trash.sql).
|
||||
-- * trigger_type:
|
||||
-- document_date -> documents.document_date, sonst created_at
|
||||
-- upload_date -> documents.created_at
|
||||
-- fixed_date -> trigger_reference als YYYY-MM-DD (einmaliger Stichtag)
|
||||
-- event -> NICHT auto-berechnet (z.B. Geschäftsjahresende /
|
||||
-- Vertragsende); Dokumente werden übersprungen, künftiger
|
||||
-- Erweiterungspunkt.
|
||||
-- * retain_until = Stichtag + retention_years Jahre + retention_days Tage.
|
||||
-- Für non-event-Regeln muss mindestens eines von years/days > 0 sein
|
||||
-- (im Go-Store validiert, nicht per CHECK).
|
||||
-- * Präzedenz: doc-typ-spezifische Regel schlägt die tenant-weite Default-
|
||||
-- Regel (doc_type_id IS NULL). UNIQUE(tenant_id, doc_type_id) erzwingt
|
||||
-- höchstens eine Regel pro (Mandant, Dokumenttyp), daher keine
|
||||
-- "strictest wins"-Logik nötig.
|
||||
-- * requires_approval_for_destroy / dsgvo_conflict sind informative Flags für
|
||||
-- das spätere Disposition-Frontend; die Vier-Augen-Pflicht selbst wird
|
||||
-- bereits vom Trash-Flow erzwungen.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS retention_rules (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
doc_type_id BIGINT REFERENCES document_types(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
trigger_type TEXT NOT NULL
|
||||
CHECK (trigger_type IN ('document_date','upload_date','fixed_date','event')),
|
||||
trigger_reference TEXT NOT NULL DEFAULT '',
|
||||
retention_years INT,
|
||||
retention_days INT,
|
||||
legal_basis TEXT NOT NULL DEFAULT '',
|
||||
requires_approval_for_destroy BOOLEAN NOT NULL DEFAULT true,
|
||||
dsgvo_conflict BOOLEAN NOT NULL DEFAULT false,
|
||||
active BOOLEAN NOT NULL DEFAULT true,
|
||||
created_by BIGINT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, doc_type_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_retention_rules_tenant ON retention_rules(tenant_id) WHERE active;
|
||||
@@ -0,0 +1,59 @@
|
||||
-- PROJ: mandanten-job-queue (Phase 1+2)
|
||||
-- Doku-only (siehe README.md in diesem Verzeichnis) — die tatsächliche
|
||||
-- Ausführung passiert idempotent über
|
||||
-- internal/storage/processing_jobs.go initProcessingJobsSchema(),
|
||||
-- aufgerufen aus (*Store).initSchema() (zuletzt in der Kette, da die Tabelle
|
||||
-- documents(id) FK-referenziert).
|
||||
--
|
||||
-- Mandanten-faire Verarbeitungs-Queue für die NACHGELAGERTE Dokument-
|
||||
-- verarbeitung (OCR-Extraktion, Taxonomie-Autozuordnung, on_upload-Workflows).
|
||||
-- Vorher lief das synchron im Upload-Request und erzeugte bei Batch-Scans /
|
||||
-- SFTP-Massenuploads Lastspitzen.
|
||||
--
|
||||
-- * Kein Redis: die Queue ist eine ganz normale Postgres-Tabelle. Dokument-
|
||||
-- INSERT und Job-INSERT laufen in EINER Transaktion
|
||||
-- (CreateDocumentWithJob) — nie ein Dokument ohne Job, nie ein Job ohne
|
||||
-- Dokument.
|
||||
-- * WORM bleibt synchron im Request: Hash, Ablage unter
|
||||
-- store/<tenant>/<yyyy>/<mm>/<sha256>.<ext> und chmod 0440 passieren VOR
|
||||
-- dem INSERT. Der Job liest die archivierte Datei nur und schreibt
|
||||
-- ausschließlich abgeleitete Metadaten.
|
||||
-- * Dispatch: Round-Robin über die Mandanten (je Runde ein Job pro Mandant),
|
||||
-- Locking über FOR UPDATE SKIP LOCKED.
|
||||
-- * Retry: exponentielles Backoff über next_attempt_at (2^retry_count
|
||||
-- Sekunden). Ab retry_count > max_retries (Default 5) bleibt der Job
|
||||
-- dauerhaft 'failed' — kein Automatik-Retry mehr.
|
||||
-- * derive_title: merkt sich, ob der Titel beim Staging nur ein Platzhalter
|
||||
-- war und aus dem OCR-Text ersetzt werden darf. Ein vom Benutzer bzw. aus
|
||||
-- dem SFTP-Dateinamen vorgegebener Titel wird nie überschrieben.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS processing_jobs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
status TEXT NOT NULL DEFAULT 'queued', -- queued | processing | done | failed
|
||||
retry_count INT NOT NULL DEFAULT 0,
|
||||
derive_title BOOLEAN NOT NULL DEFAULT false,
|
||||
error_message TEXT,
|
||||
next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
started_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
ALTER TABLE processing_jobs ADD COLUMN IF NOT EXISTS derive_title BOOLEAN NOT NULL DEFAULT false;
|
||||
ALTER TABLE processing_jobs ADD COLUMN IF NOT EXISTS next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE processing_jobs ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ;
|
||||
ALTER TABLE processing_jobs ADD COLUMN IF NOT EXISTS error_message TEXT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_processing_jobs_dispatch
|
||||
ON processing_jobs(tenant_id, status, next_attempt_at, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_processing_jobs_document
|
||||
ON processing_jobs(document_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_processing_jobs_status
|
||||
ON processing_jobs(status);
|
||||
|
||||
-- Anzeige-/Ablaufstatus am Dokument. Default 'done': der komplette Altbestand
|
||||
-- wurde noch synchron verarbeitet und ist per Definition fertig — bewusst KEIN
|
||||
-- Backfill-Skript, der Spalten-Default deckt alle Bestandszeilen ab.
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS processing_status TEXT NOT NULL DEFAULT 'done';
|
||||
@@ -0,0 +1,46 @@
|
||||
-- 024_ocr_words.sql
|
||||
-- Documentation only — applied via internal/storage/ocr_words.go
|
||||
-- (Store.initOCRWordsSchema), wired into Store.initSchema in documents.go.
|
||||
--
|
||||
-- Phase 2 of the OCR text-highlight/overlay feature (Phase 1: internal/ocr
|
||||
-- gained Result.Words []WordBox, coordinates already mapped back into the
|
||||
-- original uploaded file's coordinate space — see internal/ocr/coords.go).
|
||||
-- This migration adds the table that persists those word boxes per
|
||||
-- document, so a future Phase 3 read endpoint can serve them for a
|
||||
-- search-highlight/overlay UI without re-running OCR.
|
||||
--
|
||||
-- No tenant_id column: access is always mediated through document_id (join
|
||||
-- or verify against documents.tenant_id), never queried directly by tenant.
|
||||
--
|
||||
-- ON DELETE CASCADE on document_id: word boxes are a derived index over a
|
||||
-- document's OCR text/original file, not a GoBD document themselves, so
|
||||
-- cascading on hard-delete (trash workflow, see 007_trash.sql) is correct
|
||||
-- and does not affect WORM retention of the document's own storage_path/
|
||||
-- content_hash.
|
||||
--
|
||||
-- Every OCR (re)run (upload async job, manual reprocess endpoint, and the
|
||||
-- `documents reprocess-all` CLI, which shares ReprocessDocument with the
|
||||
-- endpoint) deletes this document's existing rows and re-inserts the fresh
|
||||
-- set (storage.ReplaceOCRWords), so re-OCR never accumulates duplicates.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ocr_words (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
page INTEGER NOT NULL DEFAULT 1,
|
||||
block INTEGER NOT NULL DEFAULT 0,
|
||||
par INTEGER NOT NULL DEFAULT 0,
|
||||
line INTEGER NOT NULL DEFAULT 0,
|
||||
word_text TEXT NOT NULL,
|
||||
"left" INTEGER NOT NULL,
|
||||
top INTEGER NOT NULL,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
confidence DOUBLE PRECISION NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_ocr_words_document ON ocr_words (document_id);
|
||||
|
||||
-- Plain btree, not pg_trgm/GIN: initSchema cannot assume CREATE EXTENSION
|
||||
-- privileges on the live server. Revisit if Phase 3 needs fuzzy/substring
|
||||
-- prefiltering (would require enabling pg_trgm out-of-band first).
|
||||
CREATE INDEX IF NOT EXISTS idx_ocr_words_word_text ON ocr_words (word_text);
|
||||
@@ -0,0 +1,31 @@
|
||||
-- 025_document_date_score.sql
|
||||
-- Documentation only — applied via internal/storage/documents.go
|
||||
-- (Store.initSchema), idempotent ALTER TABLE ADD COLUMN IF NOT EXISTS.
|
||||
--
|
||||
-- Persists the confidence score that internal/api's
|
||||
-- extractDocumentDateWithScore (and its byte-synchronous storage-package
|
||||
-- twin, documentDateFromTextWithScore in document_date.go) already computed
|
||||
-- at runtime for documents.document_date, but previously discarded — only
|
||||
-- the resulting date was ever written to the DB.
|
||||
--
|
||||
-- Purpose: quality gate for the planned Buchhaltungs-Pull-API (see
|
||||
-- MEMORY.md project_belegdatum_und_buchhaltung) — only documents whose
|
||||
-- document_date_score >= 0.75 should be automatically pullable.
|
||||
--
|
||||
-- Nullable, NUMERIC, no default and NO backfill run in this migration:
|
||||
-- existing rows have an unknown (not zero) confidence for their existing
|
||||
-- document_date, so NULL is the only correct value for them. A future
|
||||
-- backfill pass, if ever needed, would have to re-run OCR-text scoring
|
||||
-- against the stored ocr_text — deliberately out of scope here.
|
||||
--
|
||||
-- Value convention:
|
||||
-- 0.4 - 0.9 automatic keyword-proximity heuristic (see
|
||||
-- scoreForDatePosition / documentDateScoreForPosition)
|
||||
-- 1.0 manually confirmed/overridden by a user via
|
||||
-- PUT /api/documents/{id}/document-date — a manual override
|
||||
-- always replaces any prior automatic score, it is never left
|
||||
-- stale after the user's explicit correction.
|
||||
-- NULL unknown / not yet computed (pre-existing rows, or
|
||||
-- document_date itself cleared to NULL).
|
||||
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS document_date_score NUMERIC;
|
||||
@@ -0,0 +1,46 @@
|
||||
-- 026_accounting_api_keys.sql
|
||||
-- Documentation only — applied via
|
||||
-- internal/storage/accounting_api_keys.go (Store.initAccountingAPIKeysSchema),
|
||||
-- wired into storage.New() after initSharesSchema. Idempotent
|
||||
-- (CREATE TABLE / CREATE INDEX IF NOT EXISTS).
|
||||
--
|
||||
-- Per-tenant API keys for the read-only Buchhaltungs-Pull-API
|
||||
-- (GET /api/v1/accounting/documents[/{id}/file], siehe MEMORY.md
|
||||
-- project_belegdatum_und_buchhaltung). A key is a tenant-level MACHINE
|
||||
-- credential, not a user session: it grants read access to that tenant's
|
||||
-- archived documents and to nothing else, which is why creating one requires
|
||||
-- domain_admin.
|
||||
--
|
||||
-- Token handling mirrors document_shares (009_shares.sql) exactly:
|
||||
-- * raw key = "adms_" + base64url(32 crypto/rand bytes), generated once
|
||||
-- * returned to the caller EXACTLY once (POST response), never retrievable again
|
||||
-- * only the hex SHA-256 hash is persisted (key_hash, UNIQUE)
|
||||
-- * authentication always looks up by key_hash, never by id
|
||||
--
|
||||
-- Keys are never hard-deleted: revoking sets revoked_at, so the audit trail
|
||||
-- (audit event accounting_pull, Detail carries "key:<id>") stays resolvable for
|
||||
-- GoBD-Nachvollziehbarkeit. ResolveAccountingAPIKey filters revoked_at IS NULL
|
||||
-- inside the same UPDATE ... RETURNING that refreshes last_used_at, so a
|
||||
-- revoked key can never yield a tenant id.
|
||||
--
|
||||
-- No FK on tenant_id / created_by: consistent with the rest of the schema
|
||||
-- (plain BIGINT), because tenants/users are owned by other stores that
|
||||
-- initialise after storage.New().
|
||||
|
||||
CREATE TABLE IF NOT EXISTS accounting_api_keys (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
key_hash TEXT NOT NULL UNIQUE,
|
||||
label TEXT NOT NULL DEFAULT '',
|
||||
created_by BIGINT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
revoked_at TIMESTAMPTZ,
|
||||
last_used_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_accounting_api_keys_tenant ON accounting_api_keys(tenant_id);
|
||||
|
||||
-- No new columns on documents: the pull query (internal/storage/accounting_pull.go)
|
||||
-- reads existing columns only (document_date, document_date_score from
|
||||
-- 025_document_date_score.sql, doc_type_id, correspondent_id, created_at) and
|
||||
-- paginates by the (created_at, id) keyset, which idx_documents_tenant plus the
|
||||
-- primary key already support.
|
||||
@@ -0,0 +1,42 @@
|
||||
# Migrations-Konvention
|
||||
|
||||
archivdms verwendet, wie archivmail, **kein externes Migrationstool**. Das
|
||||
tatsächliche Schema wird idempotent zur Laufzeit von `initSchema()`-Funktionen
|
||||
in den jeweiligen Store-Paketen angelegt/erweitert (`CREATE TABLE IF NOT
|
||||
EXISTS`, `ALTER TABLE ... ADD COLUMN IF NOT EXISTS`, `CREATE INDEX IF NOT
|
||||
EXISTS`). Das ist sicher gegen mehrfaches Ausführen und gegen bereits
|
||||
existierende Produktiv-Datenbanken.
|
||||
|
||||
Dieses Verzeichnis dokumentiert trotzdem jede Schemaänderung als nummerierte
|
||||
`NNN_name.sql`-Datei — **rein informativ/Doku**, nicht ausführbar über ein
|
||||
Migrationstool. Sie dient als:
|
||||
|
||||
- lesbare Historie, welche Änderung wann und warum kam
|
||||
- Referenz für DBAs, die das Schema manuell nachvollziehen wollen
|
||||
- Grundlage für Code-Review von Schemaänderungen (PR zeigt sowohl den Go-Diff
|
||||
in `initSchema()` als auch die dazugehörige `NNN_name.sql`-Doku)
|
||||
|
||||
## Regeln
|
||||
|
||||
1. Jede neue Migration bekommt eine fortlaufende Nummer (`001`, `002`, ...).
|
||||
2. Der Dateiname beschreibt die Änderung kurz (`002_add_reminders.sql`).
|
||||
3. Der SQL-Inhalt muss exakt dem entsprechen, was `initSchema()` (oder das
|
||||
jeweilige Store-Paket) zur Laufzeit ausführt.
|
||||
4. Migrationen werden nie verändert oder gelöscht, nur ergänzt.
|
||||
|
||||
## Vorhandene Migrationen
|
||||
|
||||
- `001_initial.sql` — `tenants`, `users`, `token_blacklist`, `documents`,
|
||||
`audit_log`
|
||||
- `002_reminders.sql` — `reminders` (Wiedervorlage)
|
||||
- `003_documents_unique_hash.sql` — `UNIQUE INDEX (tenant_id, content_hash)` auf `documents` (DB-seitiger Duplikatschutz für die Upload-Pipeline)
|
||||
- `004_sftp_credentials.sql` — `sftp_credentials` (per-Mandant SFTP-Zugangsdaten für den eingebetteten SFTP-Server, `internal/sftpserver`)
|
||||
- `005_taxonomy.sql` — `tags`/`document_types`/`correspondents`/`document_tags` (strukturierte Entitäten mit Matching-Algorithmus + Barcode-Wert) plus `documents.doc_type_id`/`correspondent_id`/`barcode_values`-ALTER
|
||||
- `006_custom_fields.sql` — `custom_field_defs`/`document_type_fields`/`document_field_values` (benutzerdefinierte Felder pro Mandant/Dokumenttyp/Dokument)
|
||||
- `007_trash.sql` — Papierkorb + gestaffeltes Löschkonzept: `documents.deleted_at`/`deleted_by`-ALTER (Soft-Delete) plus `document_delete_requests` (Vier-Augen-Workflow für finales WORM-Löschen, Retention-Prüfung, Tombstone)
|
||||
- `011_classification_templates.sql` — `classification_templates`/`classification_template_tags`/`classification_template_field_defaults` (Klassifizierungsvorlagen: benannte Bündel aus Dokumenttyp/Tags/Custom-Field-Defaults/Aufbewahrungsdauer, anwendbar per Preview+Commit; keine persistente Kopplung an documents, Retention nur verlängerbar)
|
||||
- `021_title_template.sql` — `classification_templates.title_template`-ALTER + `tenants.default_title_template`-ALTER (Titel-Vorlage pro Klassifizierungsvorlage mit tenant-weitem Default-Fallback; Go-text/template-Rendering in `classification_templates_title.go`, angewendet in `ApplyTemplate` für manuellen Endpoint UND Workflow-Trigger; setzt Titel nur wenn `title_manually_set=false` und lässt das Flag false)
|
||||
- `022_retention_rules.sql` — `retention_rules` (GoBD-Aufbewahrungsregeln / "Disposition Schedules" pro Dokumenttyp bzw. tenant-weiter Default mit `doc_type_id IS NULL`): trigger_type (`document_date`/`upload_date`/`fixed_date`/`event`) + retention_years/days berechnen `documents.retain_until` über den Batch-Job `archivdms retention apply` (`initRetentionRulesSchema`/`ApplyRetentionRules`). Setzt nur `retain_until` (WORM-Sperre), löscht nie und verkürzt nie; die Vernichtung läuft weiter über 007_trash.sql. `event`-Regeln werden nicht auto-berechnet (künftiger Erweiterungspunkt)
|
||||
- `024_ocr_words.sql` — `ocr_words` (word-level OCR bounding boxes per Dokument, Phase 2 des Text-Highlight/Overlay-Features, `internal/storage/ocr_words.go`/`initOCRWordsSchema`): `document_id` FK `ON DELETE CASCADE`, kein `tenant_id` (immer über `document_id` mediiert), `ReplaceOCRWords` löscht+re-inserted atomar bei jedem (Re-)OCR-Lauf (Upload-Job, `/reprocess`-Endpoint, `documents reprocess-all` CLI), damit kein Duplikat-Anhäufen entsteht
|
||||
- `025_document_date_score.sql` — `documents.document_date_score` (NUMERIC, nullable, kein Backfill): persistiert die Konfidenz (0.4-0.9 automatische Keyword-Proximity-Heuristik, 1.0 = manuell bestätigt/überschrieben via `PUT /api/documents/{id}/document-date`) der bereits vorhandenen `document_date`-Erkennung, vorbereitend für die geplante Buchhaltungs-Pull-API (Score >= 0.75 als Pull-Filter)
|
||||
- `020_ml_classifier.sql` — `ml_classifier_tokens`/`ml_classifier_classes`/`ml_classifier_runs` (Naive-Bayes-Retraining-Klassifizierung Phase 1, ergänzt die Regel-Engine) plus `document_tags.assigned_via`/`documents.doc_type_assigned_via`/`documents.correspondent_assigned_via`-ALTER (Provenienz: `manual`/`rule`/`ml_accepted`, Default `manual` zum Schutz bestehender Trainingsdaten)
|
||||
@@ -0,0 +1,64 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// initMLClassifierSchema creates the Naive-Bayes retraining/classification
|
||||
// tables (ml_classifier_tokens/ml_classifier_classes/ml_classifier_runs) plus
|
||||
// the assigned_via provenance columns on document_tags/documents that let the
|
||||
// classifier tell apart manually-set, rule-engine-set and ML-accepted
|
||||
// assignments. Idempotent, called from (*Store).initSchema AFTER
|
||||
// initTaxonomySchema (depends on document_types/correspondents/tags/
|
||||
// document_tags/documents existing). See migrations/020_ml_classifier.sql.
|
||||
func (s *Store) initMLClassifierSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS ml_classifier_tokens (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('document_types','correspondents','tags')),
|
||||
entity_id BIGINT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
count BIGINT NOT NULL DEFAULT 0,
|
||||
UNIQUE (tenant_id, kind, entity_id, token)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ml_tokens_lookup ON ml_classifier_tokens(tenant_id, kind, token);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ml_classifier_classes (
|
||||
tenant_id BIGINT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
entity_id BIGINT NOT NULL,
|
||||
doc_count BIGINT NOT NULL DEFAULT 0,
|
||||
total_tokens BIGINT NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (tenant_id, kind, entity_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ml_classifier_runs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
completed_at TIMESTAMPTZ,
|
||||
doc_count BIGINT NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'running' CHECK (status IN ('running','completed','failed','skipped_insufficient_data')),
|
||||
error TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ml_classifier_runs_tenant ON ml_classifier_runs(tenant_id);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create ml_classifier tables: %w", err)
|
||||
}
|
||||
|
||||
// Provenance columns: default 'manual' so existing rows are NOT retroactively
|
||||
// (mis)classified as rule-assigned — conservative default keeps them out of
|
||||
// automatic ML-accepted reclassification too.
|
||||
_, err = s.db.Exec(ctx, `
|
||||
ALTER TABLE document_tags ADD COLUMN IF NOT EXISTS assigned_via TEXT NOT NULL DEFAULT 'manual' CHECK (assigned_via IN ('manual','rule','ml_accepted'));
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS doc_type_assigned_via TEXT NOT NULL DEFAULT 'manual' CHECK (doc_type_assigned_via IN ('manual','rule','ml_accepted'));
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS correspondent_assigned_via TEXT NOT NULL DEFAULT 'manual' CHECK (correspondent_assigned_via IN ('manual','rule','ml_accepted'));
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: alter documents/document_tags for ml_classifier provenance: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"archivdms/internal/classifier"
|
||||
)
|
||||
|
||||
// MLClassifierKinds is the fixed set of taxonomy kinds the Naive-Bayes
|
||||
// classifier is trained/predicted for, in a stable order (used by the retrain
|
||||
// CLI so runs are deterministic).
|
||||
var MLClassifierKinds = []string{
|
||||
classifier.KindDocumentTypes,
|
||||
classifier.KindCorrespondents,
|
||||
classifier.KindTags,
|
||||
}
|
||||
|
||||
// TrainClassifier rebuilds the Naive-Bayes model for one tenant and one kind and
|
||||
// returns the number of training documents used. Thin wrapper around
|
||||
// classifier.Train that keeps the Store's *pgxpool.Pool encapsulated (the
|
||||
// classifier package must not import storage). See classifier.Train for
|
||||
// semantics (full rebuild, MinDocsPerClass threshold, no error on empty data).
|
||||
func (s *Store) TrainClassifier(ctx context.Context, tenantID int64, kind string) (int, error) {
|
||||
return classifier.New(s.db).Train(ctx, tenantID, kind)
|
||||
}
|
||||
|
||||
// StartMLRun inserts a fresh ml_classifier_runs row in status 'running' for a
|
||||
// tenant and returns its id. The retrain CLI creates one run per tenant that
|
||||
// spans all kinds, then finalises it with FinishMLRun.
|
||||
func (s *Store) StartMLRun(ctx context.Context, tenantID int64) (int64, error) {
|
||||
var id int64
|
||||
if err := s.db.QueryRow(ctx,
|
||||
`INSERT INTO ml_classifier_runs (tenant_id, status) VALUES ($1, 'running') RETURNING id`,
|
||||
tenantID).Scan(&id); err != nil {
|
||||
return 0, fmt.Errorf("storage: start ml_classifier run: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// FinishMLRun finalises an ml_classifier_runs row: sets completed_at=now(),
|
||||
// doc_count, the terminal status ('completed', 'failed' or
|
||||
// 'skipped_insufficient_data') and an optional error message. errMsg "" stores
|
||||
// SQL NULL.
|
||||
func (s *Store) FinishMLRun(ctx context.Context, runID int64, docCount int, status, errMsg string) error {
|
||||
var errArg any
|
||||
if errMsg != "" {
|
||||
errArg = errMsg
|
||||
}
|
||||
_, err := s.db.Exec(ctx,
|
||||
`UPDATE ml_classifier_runs
|
||||
SET completed_at = now(), doc_count = $2, status = $3, error = $4
|
||||
WHERE id = $1`,
|
||||
runID, docCount, status, errArg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: finish ml_classifier run: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// OCRWord is one word-level bounding box persisted for a document, mirroring
|
||||
// internal/ocr.WordBox (Phase 1 of the OCR text-highlight/overlay feature —
|
||||
// see project memory project_ocr_textmarkierung_overlay.md). Coordinates are
|
||||
// already in the original, undoctored file's coordinate space (see
|
||||
// internal/ocr/coords.go package doc comment), not tesseract's
|
||||
// post-preprocessing space.
|
||||
type OCRWord struct {
|
||||
ID int64
|
||||
DocumentID int64
|
||||
Page int
|
||||
Block int
|
||||
Par int
|
||||
Line int
|
||||
Word string
|
||||
Left int
|
||||
Top int
|
||||
Width int
|
||||
Height int
|
||||
Confidence float64
|
||||
}
|
||||
|
||||
// initOCRWordsSchema creates the ocr_words table (see
|
||||
// migrations/024_ocr_words.sql). Wired into Store.initSchema after documents
|
||||
// exists (FK ON DELETE CASCADE — word boxes have no independent GoBD
|
||||
// retention meaning of their own; they are a derived index over a document's
|
||||
// ocr_text/original file, not a document themselves, so cascading their
|
||||
// deletion when the parent document row is hard-deleted from the trash
|
||||
// workflow is the correct WORM behaviour here, unlike documents.storage_path
|
||||
// or content_hash which must never be touched).
|
||||
//
|
||||
// No tenant_id column: access is always mediated through document_id, and
|
||||
// every caller (bulk insert, delete-before-reprocess, and the future read
|
||||
// endpoint in Phase 3) must join/verify against documents(tenant_id) rather
|
||||
// than filter ocr_words directly, so there is no tenant-scan risk from
|
||||
// omitting it — but see the README entry for this migration if that
|
||||
// assumption ever changes.
|
||||
func (s *Store) initOCRWordsSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS ocr_words (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
page INTEGER NOT NULL DEFAULT 1,
|
||||
block INTEGER NOT NULL DEFAULT 0,
|
||||
par INTEGER NOT NULL DEFAULT 0,
|
||||
line INTEGER NOT NULL DEFAULT 0,
|
||||
word_text TEXT NOT NULL,
|
||||
"left" INTEGER NOT NULL,
|
||||
top INTEGER NOT NULL,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
confidence DOUBLE PRECISION NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ocr_words_document ON ocr_words (document_id);
|
||||
-- Plain btree on word_text: cheap prefilter for an eventual highlight
|
||||
-- feature doing exact/prefix matching against search terms already
|
||||
-- tokenized elsewhere. Deliberately NOT a pg_trgm GIN index — pg_trgm
|
||||
-- requires "CREATE EXTENSION pg_trgm", which needs DB-superuser
|
||||
-- privileges initSchema cannot assume it has on 192.168.1.204 (see
|
||||
-- other Store init* methods: none of them create extensions). If
|
||||
-- fuzzy/substring prefiltering turns out to be needed for the Phase 3
|
||||
-- read endpoint, add the extension out-of-band on the server first,
|
||||
-- then switch this index.
|
||||
CREATE INDEX IF NOT EXISTS idx_ocr_words_word_text ON ocr_words (word_text);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create ocr_words table: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReplaceOCRWords atomically deletes any previously stored word boxes for a
|
||||
// document and bulk-inserts the new set, so reprocess/re-OCR runs never pile
|
||||
// up duplicate word rows alongside stale ones. A nil/empty words slice is
|
||||
// valid (e.g. OCR found no words, or Result.Words extraction failed
|
||||
// best-effort) and simply leaves the document with zero rows.
|
||||
func (s *Store) ReplaceOCRWords(ctx context.Context, documentID int64, words []OCRWord) error {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: begin replace ocr_words: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM ocr_words WHERE document_id = $1`, documentID); err != nil {
|
||||
return fmt.Errorf("storage: delete ocr_words: %w", err)
|
||||
}
|
||||
|
||||
for _, w := range words {
|
||||
_, err := tx.Exec(ctx, `
|
||||
INSERT INTO ocr_words
|
||||
(document_id, page, block, par, line, word_text, "left", top, width, height, confidence)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||
`, documentID, w.Page, w.Block, w.Par, w.Line, w.Word, w.Left, w.Top, w.Width, w.Height, w.Confidence)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: insert ocr_word: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("storage: commit replace ocr_words: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListOCRWords returns all persisted word boxes of a document in natural
|
||||
// reading order (page, block, par, line, then insertion order via id), which is
|
||||
// the order the overlay renderer needs to group words back into lines.
|
||||
//
|
||||
// NOTE (tenant isolation): ocr_words has no tenant_id column by design (see
|
||||
// initOCRWordsSchema), so this function is deliberately NOT tenant-scoped. Every
|
||||
// caller MUST have verified document ownership beforehand — the API handler
|
||||
// does so via GetDocument(id, tenantID), exactly like the audit/notes/file
|
||||
// sub-routes.
|
||||
//
|
||||
// Always returns a non-nil slice so JSON encoding yields [] rather than null.
|
||||
func (s *Store) ListOCRWords(ctx context.Context, documentID int64) ([]OCRWord, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, document_id, page, block, par, line, word_text, "left", top, width, height, confidence
|
||||
FROM ocr_words
|
||||
WHERE document_id = $1
|
||||
ORDER BY page, block, par, line, id
|
||||
`, documentID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list ocr_words: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]OCRWord, 0)
|
||||
for rows.Next() {
|
||||
var w OCRWord
|
||||
if err := rows.Scan(&w.ID, &w.DocumentID, &w.Page, &w.Block, &w.Par, &w.Line, &w.Word,
|
||||
&w.Left, &w.Top, &w.Width, &w.Height, &w.Confidence); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan ocr_word: %w", err)
|
||||
}
|
||||
out = append(out, w)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("storage: iterate ocr_words: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DeleteOCRWords removes all stored word boxes for a document. Exposed
|
||||
// separately from ReplaceOCRWords for callers (e.g. a future hard-delete
|
||||
// path) that need to clear word boxes without immediately re-inserting new
|
||||
// ones; ON DELETE CASCADE already handles the case where the documents row
|
||||
// itself is removed, so this is only needed when the document row survives
|
||||
// but its word boxes must be cleared independently.
|
||||
func (s *Store) DeleteOCRWords(ctx context.Context, documentID int64) error {
|
||||
if _, err := s.db.Exec(ctx, `DELETE FROM ocr_words WHERE document_id = $1`, documentID); err != nil {
|
||||
return fmt.Errorf("storage: delete ocr_words: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// OllamaConfig is the per-tenant connection configuration for an EXTERNAL,
|
||||
// already-running Ollama server (never installed on the archivdms host — the
|
||||
// URL/port is provided by the tenant admin). It gates the optional 'ollama'
|
||||
// metadata-suggestion provider. BaseURL is an internal network URL, not a
|
||||
// secret, so it is returned to the API as-is (unlike the LDAP bind password).
|
||||
type OllamaConfig struct {
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
Enabled bool `json:"enabled"`
|
||||
BaseURL string `json:"base_url"`
|
||||
Model string `json:"model"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// defaultOllamaTimeoutSeconds is used for rows that predate the column default
|
||||
// or were never configured, so GetOllamaConfig never returns a zero timeout.
|
||||
const defaultOllamaTimeoutSeconds = 30
|
||||
|
||||
// initOllamaConfigSchema creates the tenant_ollama_config table. Idempotent,
|
||||
// called from (*Store).initSchema. Documented (not executed) in
|
||||
// migrations/017_tenant_ollama_config.sql.
|
||||
//
|
||||
// No FK enforcement beyond the documented REFERENCES: consistent with the rest
|
||||
// of the schema (tenant_id is a plain BIGINT elsewhere); the PRIMARY KEY gives
|
||||
// the one-row-per-tenant upsert target.
|
||||
func (s *Store) initOllamaConfigSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS tenant_ollama_config (
|
||||
tenant_id BIGINT PRIMARY KEY,
|
||||
enabled BOOLEAN NOT NULL DEFAULT false,
|
||||
base_url TEXT NOT NULL DEFAULT '',
|
||||
model TEXT NOT NULL DEFAULT '',
|
||||
timeout_seconds INT NOT NULL DEFAULT 30,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create tenant_ollama_config table: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetOllamaConfig returns the tenant's Ollama connection config. When no row
|
||||
// exists it returns a zero/default (disabled, empty URL/model, default
|
||||
// timeout) config and NO error — callers treat "not configured" as "disabled".
|
||||
func (s *Store) GetOllamaConfig(ctx context.Context, tenantID int64) (*OllamaConfig, error) {
|
||||
row := s.db.QueryRow(ctx, `
|
||||
SELECT tenant_id, enabled, base_url, model, timeout_seconds, updated_at
|
||||
FROM tenant_ollama_config WHERE tenant_id = $1`, tenantID)
|
||||
var c OllamaConfig
|
||||
err := row.Scan(&c.TenantID, &c.Enabled, &c.BaseURL, &c.Model, &c.TimeoutSeconds, &c.UpdatedAt)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return &OllamaConfig{
|
||||
TenantID: tenantID,
|
||||
Enabled: false,
|
||||
TimeoutSeconds: defaultOllamaTimeoutSeconds,
|
||||
}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("storage: get ollama config: %w", err)
|
||||
}
|
||||
if c.TimeoutSeconds <= 0 {
|
||||
c.TimeoutSeconds = defaultOllamaTimeoutSeconds
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// UpsertOllamaConfig creates or updates the tenant's Ollama connection config.
|
||||
// When enabled is true, base_url (http/https prefixed) and model must be set
|
||||
// and timeout_seconds must be in [5,120]; when disabled the fields may be empty
|
||||
// (the switch can be flipped off without wiping the stored URL/model, but the
|
||||
// values are still range-checked when present).
|
||||
func (s *Store) UpsertOllamaConfig(ctx context.Context, tenantID int64, enabled bool, baseURL, model string, timeoutSeconds int) error {
|
||||
baseURL = strings.TrimSpace(baseURL)
|
||||
model = strings.TrimSpace(model)
|
||||
|
||||
if timeoutSeconds == 0 {
|
||||
timeoutSeconds = defaultOllamaTimeoutSeconds
|
||||
}
|
||||
if timeoutSeconds < 5 || timeoutSeconds > 120 {
|
||||
return fmt.Errorf("Timeout muss zwischen 5 und 120 Sekunden liegen")
|
||||
}
|
||||
if enabled {
|
||||
if baseURL == "" || model == "" {
|
||||
return fmt.Errorf("Server-URL und Modell müssen ausgefüllt sein, um Ollama zu aktivieren")
|
||||
}
|
||||
if !strings.HasPrefix(baseURL, "http://") && !strings.HasPrefix(baseURL, "https://") {
|
||||
return fmt.Errorf("Server-URL muss mit http:// oder https:// beginnen")
|
||||
}
|
||||
}
|
||||
|
||||
// Normalise the base URL by stripping a trailing slash so the client can
|
||||
// always append "/api/generate" without producing a double slash.
|
||||
baseURL = strings.TrimRight(baseURL, "/")
|
||||
|
||||
_, err := s.db.Exec(ctx, `
|
||||
INSERT INTO tenant_ollama_config (tenant_id, enabled, base_url, model, timeout_seconds, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, now())
|
||||
ON CONFLICT (tenant_id) DO UPDATE SET
|
||||
enabled = EXCLUDED.enabled,
|
||||
base_url = EXCLUDED.base_url,
|
||||
model = EXCLUDED.model,
|
||||
timeout_seconds = EXCLUDED.timeout_seconds,
|
||||
updated_at = now()`,
|
||||
tenantID, enabled, baseURL, model, timeoutSeconds)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: upsert ollama config: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,721 @@
|
||||
// Permission model: group-resolved, layered document ACL (see
|
||||
// migrations/008_permissions.sql). Access to a document is resolved through
|
||||
// permission_groups (never directly per-user) over three layers, most
|
||||
// specific winning:
|
||||
//
|
||||
// 1. document_grants — per-document, may 'deny' (removes a group entirely)
|
||||
// 2. tag_grants — via the document's tags
|
||||
// 3. document_type_grants — via the document's doc_type_id
|
||||
//
|
||||
// The resolved set is materialised into document_visibility by
|
||||
// RecomputeVisibility, which must be re-run whenever any grant, a document's
|
||||
// tags, or a document's doc_type changes. Roles remain the outer boundary:
|
||||
// only role 'user' is ever filtered against document_visibility;
|
||||
// domain_admin/superadmin bypass the ACL entirely (enforced in the handlers /
|
||||
// ListDocuments caller).
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
// ErrPermissionGroupNotFound is returned when a permission group lookup /
|
||||
// mutation does not match a row owned by the caller's tenant.
|
||||
var ErrPermissionGroupNotFound = errors.New("storage: permission group not found or not owned by tenant")
|
||||
|
||||
// ErrDuplicatePermissionGroup is returned on UNIQUE(tenant_id, name) violation.
|
||||
var ErrDuplicatePermissionGroup = errors.New("storage: permission group with this name already exists for tenant")
|
||||
|
||||
// ErrGrantNotFound is returned when a grant delete matches no row.
|
||||
var ErrGrantNotFound = errors.New("storage: grant not found")
|
||||
|
||||
// PermissionGroup is a named set of users, the unit ACL grants are attached to.
|
||||
type PermissionGroup struct {
|
||||
ID int64 `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
Name string `json:"name"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// initPermissionsSchema creates the permission_groups /
|
||||
// permission_group_members / *_grants / document_visibility tables. Idempotent,
|
||||
// wired in from (*Store).New. Documented in migrations/008_permissions.sql.
|
||||
func (s *Store) initPermissionsSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
-- No FK on tenant_id / user_id columns: consistent with the rest of the
|
||||
-- schema (documents/taxonomy use plain BIGINT tenant_id), and required
|
||||
-- because the tenants/users tables are created by other stores that
|
||||
-- initialise AFTER storage.New() (see cmd/archivdms/main.go ordering).
|
||||
CREATE TABLE IF NOT EXISTS permission_groups (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (tenant_id, name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS permission_group_members (
|
||||
group_id BIGINT NOT NULL REFERENCES permission_groups(id) ON DELETE CASCADE,
|
||||
user_id BIGINT NOT NULL,
|
||||
PRIMARY KEY (group_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_type_grants (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
doc_type_id BIGINT NOT NULL REFERENCES document_types(id) ON DELETE CASCADE,
|
||||
group_id BIGINT NOT NULL REFERENCES permission_groups(id) ON DELETE CASCADE,
|
||||
access TEXT NOT NULL DEFAULT 'read' CHECK (access IN ('read','write')),
|
||||
UNIQUE (doc_type_id, group_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tag_grants (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
tag_id BIGINT NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
|
||||
group_id BIGINT NOT NULL REFERENCES permission_groups(id) ON DELETE CASCADE,
|
||||
access TEXT NOT NULL DEFAULT 'read' CHECK (access IN ('read','write')),
|
||||
UNIQUE (tag_id, group_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_grants (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
group_id BIGINT NOT NULL REFERENCES permission_groups(id) ON DELETE CASCADE,
|
||||
access TEXT NOT NULL CHECK (access IN ('read','write','deny')),
|
||||
granted_by BIGINT NOT NULL,
|
||||
granted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (document_id, group_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_visibility (
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
group_id BIGINT NOT NULL REFERENCES permission_groups(id) ON DELETE CASCADE,
|
||||
access TEXT NOT NULL CHECK (access IN ('read','write')),
|
||||
PRIMARY KEY (document_id, group_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_doc_visibility_group ON document_visibility(group_id, document_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_pgm_user ON permission_group_members(user_id, group_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_permission_groups_tenant ON permission_groups(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_type_grants_type ON document_type_grants(doc_type_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_tag_grants_tag ON tag_grants(tag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_grants_doc ON document_grants(document_id);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create permissions tables: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- permission groups ---
|
||||
|
||||
// CreatePermissionGroup inserts a new group for a tenant.
|
||||
func (s *Store) CreatePermissionGroup(ctx context.Context, tenantID int64, name string) (*PermissionGroup, error) {
|
||||
var g PermissionGroup
|
||||
err := s.db.QueryRow(ctx, `
|
||||
INSERT INTO permission_groups (tenant_id, name) VALUES ($1, $2)
|
||||
RETURNING id, tenant_id, name, created_at
|
||||
`, tenantID, name).Scan(&g.ID, &g.TenantID, &g.Name, &g.CreatedAt)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return nil, ErrDuplicatePermissionGroup
|
||||
}
|
||||
return nil, fmt.Errorf("storage: create permission group: %w", err)
|
||||
}
|
||||
return &g, nil
|
||||
}
|
||||
|
||||
// ListPermissionGroups returns all groups for a tenant, name-sorted.
|
||||
func (s *Store) ListPermissionGroups(ctx context.Context, tenantID int64) ([]PermissionGroup, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, tenant_id, name, created_at FROM permission_groups
|
||||
WHERE tenant_id = $1 ORDER BY name ASC
|
||||
`, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list permission groups: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]PermissionGroup, 0)
|
||||
for rows.Next() {
|
||||
var g PermissionGroup
|
||||
if err := rows.Scan(&g.ID, &g.TenantID, &g.Name, &g.CreatedAt); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan permission group: %w", err)
|
||||
}
|
||||
out = append(out, g)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// permissionGroupExists verifies the group belongs to the tenant.
|
||||
func (s *Store) permissionGroupExists(ctx context.Context, groupID, tenantID int64) (bool, error) {
|
||||
var exists bool
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT EXISTS(SELECT 1 FROM permission_groups WHERE id = $1 AND tenant_id = $2)
|
||||
`, groupID, tenantID).Scan(&exists)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("storage: check permission group: %w", err)
|
||||
}
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
// DeletePermissionGroup removes a group (cascades to members/grants/visibility),
|
||||
// scoped to tenant ownership.
|
||||
func (s *Store) DeletePermissionGroup(ctx context.Context, groupID, tenantID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `DELETE FROM permission_groups WHERE id = $1 AND tenant_id = $2`, groupID, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: delete permission group: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrPermissionGroupNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- group membership ---
|
||||
|
||||
// AddGroupMember adds a user to a group, verifying both belong to the tenant.
|
||||
// Idempotent (ON CONFLICT DO NOTHING).
|
||||
func (s *Store) AddGroupMember(ctx context.Context, groupID, userID, tenantID int64) error {
|
||||
ok, err := s.permissionGroupExists(ctx, groupID, tenantID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return ErrPermissionGroupNotFound
|
||||
}
|
||||
var userOK bool
|
||||
if err := s.db.QueryRow(ctx, `
|
||||
SELECT EXISTS(SELECT 1 FROM users WHERE id = $1 AND tenant_id = $2)
|
||||
`, userID, tenantID).Scan(&userOK); err != nil {
|
||||
return fmt.Errorf("storage: check user tenant: %w", err)
|
||||
}
|
||||
if !userOK {
|
||||
return ErrPermissionGroupNotFound
|
||||
}
|
||||
_, err = s.db.Exec(ctx, `
|
||||
INSERT INTO permission_group_members (group_id, user_id) VALUES ($1, $2)
|
||||
ON CONFLICT (group_id, user_id) DO NOTHING
|
||||
`, groupID, userID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: add group member: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveGroupMember removes a user from a group, scoped to tenant ownership of
|
||||
// the group.
|
||||
func (s *Store) RemoveGroupMember(ctx context.Context, groupID, userID, tenantID int64) error {
|
||||
ok, err := s.permissionGroupExists(ctx, groupID, tenantID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return ErrPermissionGroupNotFound
|
||||
}
|
||||
_, err = s.db.Exec(ctx, `DELETE FROM permission_group_members WHERE group_id = $1 AND user_id = $2`, groupID, userID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: remove group member: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListGroupMembers returns the user IDs in a group, scoped to tenant.
|
||||
func (s *Store) ListGroupMembers(ctx context.Context, groupID, tenantID int64) ([]int64, error) {
|
||||
ok, err := s.permissionGroupExists(ctx, groupID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
return nil, ErrPermissionGroupNotFound
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `SELECT user_id FROM permission_group_members WHERE group_id = $1 ORDER BY user_id`, groupID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list group members: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []int64
|
||||
for rows.Next() {
|
||||
var uid int64
|
||||
if err := rows.Scan(&uid); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan group member: %w", err)
|
||||
}
|
||||
out = append(out, uid)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// GroupMember is a user enriched from the users table, as returned by
|
||||
// ListGroupMembersDetailed for the group-administration UI.
|
||||
type GroupMember struct {
|
||||
UserID int64 `json:"user_id"`
|
||||
Username string `json:"username"`
|
||||
Email string `json:"email"`
|
||||
}
|
||||
|
||||
// ListGroupMembersDetailed returns the members of a group joined against the
|
||||
// users table (user_id, username, email), scoped to tenant ownership of the
|
||||
// group. Returns an empty slice (not nil) for an empty group.
|
||||
func (s *Store) ListGroupMembersDetailed(ctx context.Context, groupID, tenantID int64) ([]GroupMember, error) {
|
||||
ok, err := s.permissionGroupExists(ctx, groupID, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
return nil, ErrPermissionGroupNotFound
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT u.id, u.username, u.email
|
||||
FROM permission_group_members pgm
|
||||
JOIN users u ON u.id = pgm.user_id
|
||||
WHERE pgm.group_id = $1 AND u.tenant_id = $2
|
||||
ORDER BY u.username ASC
|
||||
`, groupID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list group members detailed: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []GroupMember{}
|
||||
for rows.Next() {
|
||||
var m GroupMember
|
||||
if err := rows.Scan(&m.UserID, &m.Username, &m.Email); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan group member detailed: %w", err)
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// GrantInfo is a grant enriched with the group name, as returned by the
|
||||
// grant-listing endpoints for the permission-administration UI.
|
||||
type GrantInfo struct {
|
||||
GroupID int64 `json:"group_id"`
|
||||
GroupName string `json:"group_name"`
|
||||
Access string `json:"access"`
|
||||
}
|
||||
|
||||
// ListDocumentTypeGrants returns the grants on a document type (group_id,
|
||||
// group_name, access), scoped to tenant. Verifies type ownership.
|
||||
func (s *Store) ListDocumentTypeGrants(ctx context.Context, tenantID, docTypeID int64) ([]GrantInfo, error) {
|
||||
var ok bool
|
||||
if err := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM document_types WHERE id = $1 AND tenant_id = $2)`, docTypeID, tenantID).Scan(&ok); err != nil {
|
||||
return nil, fmt.Errorf("storage: check document type: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return nil, ErrPermissionGroupNotFound
|
||||
}
|
||||
return s.listGrantInfos(ctx, `
|
||||
SELECT dtg.group_id, pg.name, dtg.access
|
||||
FROM document_type_grants dtg
|
||||
JOIN permission_groups pg ON pg.id = dtg.group_id
|
||||
WHERE dtg.tenant_id = $1 AND dtg.doc_type_id = $2
|
||||
ORDER BY pg.name ASC
|
||||
`, tenantID, docTypeID)
|
||||
}
|
||||
|
||||
// ListTagGrants returns the grants on a tag (group_id, group_name, access),
|
||||
// scoped to tenant. Verifies tag ownership.
|
||||
func (s *Store) ListTagGrants(ctx context.Context, tenantID, tagID int64) ([]GrantInfo, error) {
|
||||
var ok bool
|
||||
if err := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM tags WHERE id = $1 AND tenant_id = $2)`, tagID, tenantID).Scan(&ok); err != nil {
|
||||
return nil, fmt.Errorf("storage: check tag: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return nil, ErrPermissionGroupNotFound
|
||||
}
|
||||
return s.listGrantInfos(ctx, `
|
||||
SELECT tg.group_id, pg.name, tg.access
|
||||
FROM tag_grants tg
|
||||
JOIN permission_groups pg ON pg.id = tg.group_id
|
||||
WHERE tg.tenant_id = $1 AND tg.tag_id = $2
|
||||
ORDER BY pg.name ASC
|
||||
`, tenantID, tagID)
|
||||
}
|
||||
|
||||
// ListDocumentGrants returns the per-document grants (group_id, group_name,
|
||||
// access — may be 'deny'), scoped to tenant. Verifies document ownership.
|
||||
func (s *Store) ListDocumentGrants(ctx context.Context, tenantID, documentID int64) ([]GrantInfo, error) {
|
||||
var ok bool
|
||||
if err := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM documents WHERE id = $1 AND tenant_id = $2)`, documentID, tenantID).Scan(&ok); err != nil {
|
||||
return nil, fmt.Errorf("storage: check document: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return nil, ErrPermissionGroupNotFound
|
||||
}
|
||||
return s.listGrantInfos(ctx, `
|
||||
SELECT dg.group_id, pg.name, dg.access
|
||||
FROM document_grants dg
|
||||
JOIN permission_groups pg ON pg.id = dg.group_id
|
||||
WHERE dg.tenant_id = $1 AND dg.document_id = $2
|
||||
ORDER BY pg.name ASC
|
||||
`, tenantID, documentID)
|
||||
}
|
||||
|
||||
// listGrantInfos runs a (group_id, group_name, access) query and scans it into
|
||||
// a non-nil GrantInfo slice.
|
||||
func (s *Store) listGrantInfos(ctx context.Context, query string, args ...any) ([]GrantInfo, error) {
|
||||
rows, err := s.db.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list grants: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []GrantInfo{}
|
||||
for rows.Next() {
|
||||
var g GrantInfo
|
||||
if err := rows.Scan(&g.GroupID, &g.GroupName, &g.Access); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan grant info: %w", err)
|
||||
}
|
||||
out = append(out, g)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListGroupIDsForUser returns the permission-group ids a user belongs to,
|
||||
// scoped to the tenant (a group is only counted when it is owned by the
|
||||
// tenant). Used by the search endpoint to build the ANY(acl_group_ids) filter
|
||||
// for role 'user' (domain_admin/superadmin never call this — they bypass the
|
||||
// ACL). Returns an empty slice (not nil) when the user is in no group, so the
|
||||
// caller can distinguish "no ACL filter" (nil) from "sees nothing" (empty).
|
||||
func (s *Store) ListGroupIDsForUser(ctx context.Context, userID, tenantID int64) ([]int64, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT pgm.group_id
|
||||
FROM permission_group_members pgm
|
||||
JOIN permission_groups pg ON pg.id = pgm.group_id
|
||||
WHERE pgm.user_id = $1 AND pg.tenant_id = $2
|
||||
ORDER BY pgm.group_id
|
||||
`, userID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list group ids for user: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []int64{}
|
||||
for rows.Next() {
|
||||
var gid int64
|
||||
if err := rows.Scan(&gid); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan group id: %w", err)
|
||||
}
|
||||
out = append(out, gid)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// --- document-type grants ---
|
||||
|
||||
// SetDocumentTypeGrant upserts a document-type default grant and recomputes
|
||||
// visibility for every document of that type. Verifies type + group tenant.
|
||||
func (s *Store) SetDocumentTypeGrant(ctx context.Context, tenantID, docTypeID, groupID int64, access string) error {
|
||||
if access != "read" && access != "write" {
|
||||
return fmt.Errorf("storage: invalid access %q", access)
|
||||
}
|
||||
if err := s.checkTypeAndGroup(ctx, tenantID, docTypeID, groupID); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := s.db.Exec(ctx, `
|
||||
INSERT INTO document_type_grants (tenant_id, doc_type_id, group_id, access)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (doc_type_id, group_id) DO UPDATE SET access = EXCLUDED.access
|
||||
`, tenantID, docTypeID, groupID, access)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: set document type grant: %w", err)
|
||||
}
|
||||
return s.recomputeVisibilityForDocType(ctx, tenantID, docTypeID)
|
||||
}
|
||||
|
||||
// DeleteDocumentTypeGrant removes a document-type grant and recomputes
|
||||
// visibility for that type's documents.
|
||||
func (s *Store) DeleteDocumentTypeGrant(ctx context.Context, tenantID, docTypeID, groupID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
DELETE FROM document_type_grants WHERE tenant_id = $1 AND doc_type_id = $2 AND group_id = $3
|
||||
`, tenantID, docTypeID, groupID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: delete document type grant: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrGrantNotFound
|
||||
}
|
||||
return s.recomputeVisibilityForDocType(ctx, tenantID, docTypeID)
|
||||
}
|
||||
|
||||
// --- tag grants ---
|
||||
|
||||
// SetTagGrant upserts a tag grant and recomputes visibility for every document
|
||||
// carrying that tag. Verifies tag + group tenant.
|
||||
func (s *Store) SetTagGrant(ctx context.Context, tenantID, tagID, groupID int64, access string) error {
|
||||
if access != "read" && access != "write" {
|
||||
return fmt.Errorf("storage: invalid access %q", access)
|
||||
}
|
||||
if err := s.checkTagAndGroup(ctx, tenantID, tagID, groupID); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := s.db.Exec(ctx, `
|
||||
INSERT INTO tag_grants (tenant_id, tag_id, group_id, access)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (tag_id, group_id) DO UPDATE SET access = EXCLUDED.access
|
||||
`, tenantID, tagID, groupID, access)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: set tag grant: %w", err)
|
||||
}
|
||||
return s.recomputeVisibilityForTag(ctx, tagID)
|
||||
}
|
||||
|
||||
// DeleteTagGrant removes a tag grant and recomputes visibility for that tag's
|
||||
// documents.
|
||||
func (s *Store) DeleteTagGrant(ctx context.Context, tenantID, tagID, groupID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
DELETE FROM tag_grants WHERE tenant_id = $1 AND tag_id = $2 AND group_id = $3
|
||||
`, tenantID, tagID, groupID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: delete tag grant: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrGrantNotFound
|
||||
}
|
||||
return s.recomputeVisibilityForTag(ctx, tagID)
|
||||
}
|
||||
|
||||
// --- document grants ---
|
||||
|
||||
// SetDocumentGrant upserts a per-document grant ('read'/'write'/'deny') and
|
||||
// recomputes visibility for that document. Verifies document + group tenant.
|
||||
func (s *Store) SetDocumentGrant(ctx context.Context, tenantID, documentID, groupID, grantedBy int64, access string) error {
|
||||
if access != "read" && access != "write" && access != "deny" {
|
||||
return fmt.Errorf("storage: invalid access %q", access)
|
||||
}
|
||||
if err := s.checkDocAndGroup(ctx, tenantID, documentID, groupID); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := s.db.Exec(ctx, `
|
||||
INSERT INTO document_grants (tenant_id, document_id, group_id, access, granted_by)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (document_id, group_id) DO UPDATE SET access = EXCLUDED.access, granted_by = EXCLUDED.granted_by, granted_at = now()
|
||||
`, tenantID, documentID, groupID, access, grantedBy)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: set document grant: %w", err)
|
||||
}
|
||||
return s.RecomputeVisibility(ctx, documentID)
|
||||
}
|
||||
|
||||
// DeleteDocumentGrant removes a per-document grant and recomputes visibility.
|
||||
func (s *Store) DeleteDocumentGrant(ctx context.Context, tenantID, documentID, groupID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
DELETE FROM document_grants WHERE tenant_id = $1 AND document_id = $2 AND group_id = $3
|
||||
`, tenantID, documentID, groupID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: delete document grant: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrGrantNotFound
|
||||
}
|
||||
return s.RecomputeVisibility(ctx, documentID)
|
||||
}
|
||||
|
||||
// --- tenant/ownership checks ---
|
||||
|
||||
func (s *Store) checkTypeAndGroup(ctx context.Context, tenantID, docTypeID, groupID int64) error {
|
||||
var ok bool
|
||||
if err := s.db.QueryRow(ctx, `
|
||||
SELECT EXISTS(SELECT 1 FROM document_types WHERE id = $1 AND tenant_id = $2)
|
||||
AND EXISTS(SELECT 1 FROM permission_groups WHERE id = $3 AND tenant_id = $2)
|
||||
`, docTypeID, tenantID, groupID).Scan(&ok); err != nil {
|
||||
return fmt.Errorf("storage: check type/group: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return ErrPermissionGroupNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) checkTagAndGroup(ctx context.Context, tenantID, tagID, groupID int64) error {
|
||||
var ok bool
|
||||
if err := s.db.QueryRow(ctx, `
|
||||
SELECT EXISTS(SELECT 1 FROM tags WHERE id = $1 AND tenant_id = $2)
|
||||
AND EXISTS(SELECT 1 FROM permission_groups WHERE id = $3 AND tenant_id = $2)
|
||||
`, tagID, tenantID, groupID).Scan(&ok); err != nil {
|
||||
return fmt.Errorf("storage: check tag/group: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return ErrPermissionGroupNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) checkDocAndGroup(ctx context.Context, tenantID, documentID, groupID int64) error {
|
||||
var ok bool
|
||||
if err := s.db.QueryRow(ctx, `
|
||||
SELECT EXISTS(SELECT 1 FROM documents WHERE id = $1 AND tenant_id = $2)
|
||||
AND EXISTS(SELECT 1 FROM permission_groups WHERE id = $3 AND tenant_id = $2)
|
||||
`, documentID, tenantID, groupID).Scan(&ok); err != nil {
|
||||
return fmt.Errorf("storage: check doc/group: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return ErrPermissionGroupNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- visibility recomputation ---
|
||||
|
||||
// recomputeVisibilityForDocType recomputes visibility for all documents of a
|
||||
// given document type in a tenant.
|
||||
func (s *Store) recomputeVisibilityForDocType(ctx context.Context, tenantID, docTypeID int64) error {
|
||||
ids, err := s.collectDocIDs(ctx, `SELECT id FROM documents WHERE tenant_id = $1 AND doc_type_id = $2`, tenantID, docTypeID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.recomputeMany(ctx, ids)
|
||||
}
|
||||
|
||||
// recomputeVisibilityForTag recomputes visibility for all documents carrying a
|
||||
// given tag.
|
||||
func (s *Store) recomputeVisibilityForTag(ctx context.Context, tagID int64) error {
|
||||
ids, err := s.collectDocIDs(ctx, `SELECT document_id FROM document_tags WHERE tag_id = $1`, tagID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.recomputeMany(ctx, ids)
|
||||
}
|
||||
|
||||
func (s *Store) collectDocIDs(ctx context.Context, query string, args ...any) ([]int64, error) {
|
||||
rows, err := s.db.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: collect document ids: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var ids []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan document id: %w", err)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) recomputeMany(ctx context.Context, ids []int64) error {
|
||||
for _, id := range ids {
|
||||
if err := s.RecomputeVisibility(ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecomputeVisibility rebuilds document_visibility for a single document by
|
||||
// resolving the three grant layers (document > tag > doc_type). A
|
||||
// document_grant with access 'deny' removes that group from every lower layer.
|
||||
// DELETE+INSERT run in one transaction so a document's visibility is never
|
||||
// observed half-written.
|
||||
func (s *Store) RecomputeVisibility(ctx context.Context, documentID int64) error {
|
||||
// Layer 1: per-document grants (highest priority). 'deny' blocks a group.
|
||||
docGrants, err := s.readGrants(ctx, `SELECT group_id, access FROM document_grants WHERE document_id = $1`, documentID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
denied := make(map[int64]bool)
|
||||
resolved := make(map[int64]string)
|
||||
for _, g := range docGrants {
|
||||
if g.access == "deny" {
|
||||
denied[g.group] = true
|
||||
continue
|
||||
}
|
||||
resolved[g.group] = g.access
|
||||
}
|
||||
|
||||
// Layer 2: tag grants, via the document's tags.
|
||||
tagGrants, err := s.readGrants(ctx, `
|
||||
SELECT tg.group_id, tg.access
|
||||
FROM tag_grants tg
|
||||
JOIN document_tags dt ON dt.tag_id = tg.tag_id
|
||||
WHERE dt.document_id = $1
|
||||
`, documentID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, g := range tagGrants {
|
||||
applyLayer(resolved, denied, g.group, g.access)
|
||||
}
|
||||
|
||||
// Layer 3: document-type default grants.
|
||||
typeGrants, err := s.readGrants(ctx, `
|
||||
SELECT dtg.group_id, dtg.access
|
||||
FROM document_type_grants dtg
|
||||
JOIN documents d ON d.doc_type_id = dtg.doc_type_id
|
||||
WHERE d.id = $1
|
||||
`, documentID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, g := range typeGrants {
|
||||
applyLayer(resolved, denied, g.group, g.access)
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: begin recompute visibility: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM document_visibility WHERE document_id = $1`, documentID); err != nil {
|
||||
return fmt.Errorf("storage: clear document visibility: %w", err)
|
||||
}
|
||||
for group, access := range resolved {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO document_visibility (document_id, group_id, access) VALUES ($1, $2, $3)
|
||||
`, documentID, group, access); err != nil {
|
||||
return fmt.Errorf("storage: insert document visibility: %w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("storage: commit recompute visibility: %w", err)
|
||||
}
|
||||
// The materialised ACL (and, when reached via AttachTag/DetachTag/
|
||||
// SetDocumentDocType, the tags/doc_type) just changed — re-sync the search
|
||||
// index. Best-effort: never fails the caller.
|
||||
s.SyncIndex(ctx, documentID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyLayer adds a lower-priority grant only when the group is neither denied
|
||||
// nor already resolved by a higher layer. Within a layer, 'write' beats 'read'.
|
||||
func applyLayer(resolved map[int64]string, denied map[int64]bool, group int64, access string) {
|
||||
if denied[group] {
|
||||
return
|
||||
}
|
||||
if existing, ok := resolved[group]; ok {
|
||||
if existing == "write" || access != "write" {
|
||||
return
|
||||
}
|
||||
}
|
||||
resolved[group] = access
|
||||
}
|
||||
|
||||
type grantRow struct {
|
||||
group int64
|
||||
access string
|
||||
}
|
||||
|
||||
func (s *Store) readGrants(ctx context.Context, query string, args ...any) ([]grantRow, error) {
|
||||
rows, err := s.db.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: read grants: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []grantRow
|
||||
for rows.Next() {
|
||||
var g grantRow
|
||||
if err := rows.Scan(&g.group, &g.access); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan grant: %w", err)
|
||||
}
|
||||
out = append(out, g)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
// --- Mandanten-Job-Queue (asynchrone Dokumentverarbeitung) -----------------
|
||||
//
|
||||
// processing_jobs ist die Postgres-basierte Arbeitswarteschlange für die
|
||||
// NACHGELAGERTE Dokumentverarbeitung (OCR-Extraktion, Taxonomie-
|
||||
// Autozuordnung, on_upload-Workflows). Bewusst KEIN Redis: die Queue ist
|
||||
// Teil derselben transaktionalen Einheit wie der documents-INSERT, damit ein
|
||||
// Dokument niemals ohne Job (verwaist, ewig "queued") und ein Job niemals
|
||||
// ohne Dokument existieren kann — siehe CreateDocumentWithJob.
|
||||
//
|
||||
// WORM bleibt synchron: Datei-Hashing, Move nach store/ und chmod 0440
|
||||
// passieren VOR dem Insert im Request-Pfad (internal/api stageDocument). Der
|
||||
// Job fasst die archivierte Datei nur lesend an und schreibt ausschließlich
|
||||
// abgeleitete Metadaten (ocr_text, document_date, Titel, Tags/Typ, Workflow-
|
||||
// Ergebnisse).
|
||||
//
|
||||
// Fairness: der Dispatcher (internal/jobqueue) zieht pro Tick EINEN Job je
|
||||
// Mandant (ClaimNextJobForTenant) statt global FIFO, damit ein Massen-Upload
|
||||
// eines Mandanten die Verarbeitung aller anderen Mandanten nicht blockiert.
|
||||
// Das Locking läuft über FOR UPDATE SKIP LOCKED, dadurch sind beliebig viele
|
||||
// Worker-Goroutinen (und theoretisch auch mehrere Prozesse) kollisionsfrei.
|
||||
|
||||
// Job-Status-Werte (processing_jobs.status und documents.processing_status).
|
||||
const (
|
||||
JobStatusQueued = "queued"
|
||||
JobStatusProcessing = "processing"
|
||||
JobStatusDone = "done"
|
||||
JobStatusFailed = "failed"
|
||||
)
|
||||
|
||||
// ErrNoJob wird von ClaimNextJobForTenant zurückgegeben, wenn für den
|
||||
// Mandanten aktuell kein fälliger Job in der Queue liegt. Kein Fehlerfall —
|
||||
// der Dispatcher behandelt das als "Mandant hat nichts zu tun".
|
||||
var ErrNoJob = errors.New("storage: no queued job available")
|
||||
|
||||
// ProcessingJob ist ein Eintrag der Verarbeitungswarteschlange.
|
||||
type ProcessingJob struct {
|
||||
ID int64 `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
DocumentID int64 `json:"document_id"`
|
||||
Status string `json:"status"`
|
||||
RetryCount int `json:"retry_count"`
|
||||
// DeriveTitle merkt sich, ob der Titel beim Staging nur ein Platzhalter
|
||||
// war ("Scan 30.07.2026") und daher aus dem OCR-Text neu abgeleitet werden
|
||||
// darf. Bei einem vom Benutzer/SFTP-Dateinamen vorgegebenen Titel ist das
|
||||
// false, damit die asynchrone Verarbeitung eine bewusste Vorgabe niemals
|
||||
// überschreibt (entspricht dem alten synchronen Verhalten: Titel != "" ->
|
||||
// OCR-Ableitung wurde übersprungen).
|
||||
DeriveTitle bool `json:"derive_title"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
NextAttemptAt time.Time `json:"next_attempt_at"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// initProcessingJobsSchema legt die Queue-Tabelle an und ergänzt
|
||||
// documents.processing_status. Idempotent, aus (*Store).initSchema()
|
||||
// aufgerufen — siehe migrations/023_processing_jobs.sql (Doku).
|
||||
func (s *Store) initProcessingJobsSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS processing_jobs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
retry_count INT NOT NULL DEFAULT 0,
|
||||
derive_title BOOLEAN NOT NULL DEFAULT false,
|
||||
error_message TEXT,
|
||||
next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
started_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create processing_jobs table: %w", err)
|
||||
}
|
||||
|
||||
// Nachträglich ergänzte Spalten (idempotent, für Bestands-Installationen
|
||||
// die eine frühere Variante der Tabelle haben).
|
||||
_, err = s.db.Exec(ctx, `
|
||||
ALTER TABLE processing_jobs ADD COLUMN IF NOT EXISTS derive_title BOOLEAN NOT NULL DEFAULT false;
|
||||
ALTER TABLE processing_jobs ADD COLUMN IF NOT EXISTS next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now();
|
||||
ALTER TABLE processing_jobs ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ;
|
||||
ALTER TABLE processing_jobs ADD COLUMN IF NOT EXISTS error_message TEXT;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: alter processing_jobs table: %w", err)
|
||||
}
|
||||
|
||||
// Dispatch-Index: der Claim-Pfad filtert immer auf
|
||||
// (tenant_id, status, next_attempt_at) und sortiert nach created_at.
|
||||
_, err = s.db.Exec(ctx, `
|
||||
CREATE INDEX IF NOT EXISTS idx_processing_jobs_dispatch
|
||||
ON processing_jobs(tenant_id, status, next_attempt_at, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_processing_jobs_document
|
||||
ON processing_jobs(document_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_processing_jobs_status
|
||||
ON processing_jobs(status);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create processing_jobs indexes: %w", err)
|
||||
}
|
||||
|
||||
// processing_status auf documents: Default 'done' — der komplette
|
||||
// Altbestand wurde noch synchron im Request verarbeitet und ist damit
|
||||
// per Definition fertig. Bewusst KEIN Backfill-Migrationscode, der
|
||||
// Spalten-Default deckt alle Bestandszeilen ab.
|
||||
_, err = s.db.Exec(ctx, `
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS processing_status TEXT NOT NULL DEFAULT 'done';
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: alter documents add processing_status: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateDocumentWithJob legt Dokument UND Verarbeitungsjob in EINER
|
||||
// Transaktion an. Entweder beides oder nichts — ein Dokument ohne Job würde
|
||||
// nie OCR/Taxonomie bekommen, ein Job ohne Dokument liefe ins Leere.
|
||||
//
|
||||
// Das Dokument wird mit processing_status='queued' angelegt; der Worker setzt
|
||||
// es später auf 'done' bzw. 'failed'.
|
||||
func (s *Store) CreateDocumentWithJob(ctx context.Context, req CreateDocumentRequest, deriveTitle bool) (*Document, *ProcessingJob, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("storage: begin create document tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var d Document
|
||||
err = tx.QueryRow(ctx, `
|
||||
INSERT INTO documents (tenant_id, title, doc_type, correspondent, storage_path, content_hash, ocr_text, retain_until, document_date, source, source_ref, created_by, processing_status)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, 'queued')
|
||||
RETURNING id, tenant_id, title, COALESCE(doc_type, ''), COALESCE(correspondent, ''), doc_type_id, correspondent_id, storage_path, content_hash,
|
||||
COALESCE(ocr_text, ''), retain_until, document_date, COALESCE(source, ''), COALESCE(source_ref, ''), created_by, title_manually_set, has_thumbnail, processing_status, created_at, updated_at
|
||||
`, req.TenantID, req.Title, nullIfEmpty(req.DocType), nullIfEmpty(req.Correspondent), req.StoragePath, req.ContentHash,
|
||||
nullIfEmpty(req.OCRText), req.RetainUntil, req.DocumentDate, nullIfEmpty(req.Source), nullIfEmpty(req.SourceRef), req.CreatedBy,
|
||||
).Scan(&d.ID, &d.TenantID, &d.Title, &d.DocType, &d.Correspondent, &d.DocTypeID, &d.CorrespondentID, &d.StoragePath, &d.ContentHash,
|
||||
&d.OCRText, &d.RetainUntil, &d.DocumentDate, &d.Source, &d.SourceRef, &d.CreatedBy, &d.TitleManuallySet, &d.HasThumbnail, &d.ProcessingStatus, &d.CreatedAt, &d.UpdatedAt)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return nil, nil, ErrDuplicateContentHash
|
||||
}
|
||||
return nil, nil, fmt.Errorf("storage: create document: %w", err)
|
||||
}
|
||||
|
||||
var j ProcessingJob
|
||||
err = tx.QueryRow(ctx, `
|
||||
INSERT INTO processing_jobs (tenant_id, document_id, status, derive_title)
|
||||
VALUES ($1, $2, 'queued', $3)
|
||||
RETURNING id, tenant_id, document_id, status, retry_count, derive_title, COALESCE(error_message, ''), next_attempt_at, started_at, created_at, updated_at
|
||||
`, req.TenantID, d.ID, deriveTitle).Scan(&j.ID, &j.TenantID, &j.DocumentID, &j.Status, &j.RetryCount, &j.DeriveTitle,
|
||||
&j.ErrorMessage, &j.NextAttemptAt, &j.StartedAt, &j.CreatedAt, &j.UpdatedAt)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("storage: create processing job: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, nil, fmt.Errorf("storage: commit create document tx: %w", err)
|
||||
}
|
||||
return &d, &j, nil
|
||||
}
|
||||
|
||||
// TenantsWithDueJobs liefert alle Mandanten-IDs, für die aktuell mindestens
|
||||
// ein fälliger (queued, next_attempt_at <= now()) Job in der Queue liegt.
|
||||
// Grundlage des Round-Robin-Dispatch: der Dispatcher iteriert über diese
|
||||
// Liste und zieht je Mandant genau einen Job pro Runde.
|
||||
func (s *Store) TenantsWithDueJobs(ctx context.Context) ([]int64, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT DISTINCT tenant_id FROM processing_jobs
|
||||
WHERE status = 'queued' AND next_attempt_at <= now()
|
||||
ORDER BY tenant_id
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list tenants with due jobs: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]int64, 0)
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan tenant with due jobs: %w", err)
|
||||
}
|
||||
out = append(out, id)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimNextJobForTenant markiert den ältesten fälligen Job eines Mandanten
|
||||
// atomar als 'processing' und gibt ihn zurück. FOR UPDATE SKIP LOCKED sorgt
|
||||
// dafür, dass parallele Worker sich nicht gegenseitig blockieren und kein Job
|
||||
// doppelt gezogen wird. Liefert ErrNoJob, wenn nichts anliegt.
|
||||
func (s *Store) ClaimNextJobForTenant(ctx context.Context, tenantID int64) (*ProcessingJob, error) {
|
||||
var j ProcessingJob
|
||||
err := s.db.QueryRow(ctx, `
|
||||
UPDATE processing_jobs SET status = 'processing', started_at = now(), updated_at = now()
|
||||
WHERE id = (
|
||||
SELECT id FROM processing_jobs
|
||||
WHERE tenant_id = $1 AND status = 'queued' AND next_attempt_at <= now()
|
||||
ORDER BY created_at
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
RETURNING id, tenant_id, document_id, status, retry_count, derive_title, COALESCE(error_message, ''), next_attempt_at, started_at, created_at, updated_at
|
||||
`, tenantID).Scan(&j.ID, &j.TenantID, &j.DocumentID, &j.Status, &j.RetryCount, &j.DeriveTitle,
|
||||
&j.ErrorMessage, &j.NextAttemptAt, &j.StartedAt, &j.CreatedAt, &j.UpdatedAt)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNoJob
|
||||
}
|
||||
return nil, fmt.Errorf("storage: claim processing job: %w", err)
|
||||
}
|
||||
// Dokument-Status mitziehen, damit die UI "wird verarbeitet" anzeigen kann.
|
||||
// Best-effort: der Job ist bereits verbindlich geclaimt, ein Fehler beim
|
||||
// reinen Anzeigestatus darf die Verarbeitung nicht verhindern.
|
||||
_ = s.SetDocumentProcessingStatus(ctx, j.DocumentID, j.TenantID, JobStatusProcessing)
|
||||
return &j, nil
|
||||
}
|
||||
|
||||
// MarkJobDone schließt einen Job erfolgreich ab und setzt das Dokument auf
|
||||
// processing_status='done'.
|
||||
func (s *Store) MarkJobDone(ctx context.Context, jobID, tenantID, documentID int64) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
UPDATE processing_jobs SET status = 'done', error_message = NULL, updated_at = now()
|
||||
WHERE id = $1 AND tenant_id = $2
|
||||
`, jobID, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: mark job done: %w", err)
|
||||
}
|
||||
if err := s.SetDocumentProcessingStatus(ctx, documentID, tenantID, JobStatusDone); err != nil && !errors.Is(err, ErrDocumentNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarkJobFailed verbucht einen fehlgeschlagenen Versuch.
|
||||
//
|
||||
// Solange retry_count < maxRetries geht der Job zurück auf 'queued' mit
|
||||
// exponentiellem Backoff (2^retry_count Sekunden ab jetzt). Ist das Limit
|
||||
// erreicht, bleibt er dauerhaft auf 'failed' — kein weiterer Automatik-Retry,
|
||||
// ein manueller Retry (Phase 3, Frontend-Button) muss ihn explizit
|
||||
// requeuen (RequeueJob).
|
||||
//
|
||||
// Rückgabewert requeued sagt dem Aufrufer, ob noch ein Versuch folgt (für
|
||||
// Logging/Audit-Detail).
|
||||
func (s *Store) MarkJobFailed(ctx context.Context, jobID, tenantID, documentID int64, jobErr string, maxRetries int) (requeued bool, err error) {
|
||||
var retryCount int
|
||||
err = s.db.QueryRow(ctx, `
|
||||
SELECT retry_count FROM processing_jobs WHERE id = $1 AND tenant_id = $2
|
||||
`, jobID, tenantID).Scan(&retryCount)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("storage: load job retry_count: %w", err)
|
||||
}
|
||||
|
||||
next := retryCount + 1
|
||||
if next > maxRetries {
|
||||
if _, err := s.db.Exec(ctx, `
|
||||
UPDATE processing_jobs SET status = 'failed', retry_count = $3, error_message = $4, updated_at = now()
|
||||
WHERE id = $1 AND tenant_id = $2
|
||||
`, jobID, tenantID, next, jobErr); err != nil {
|
||||
return false, fmt.Errorf("storage: mark job failed: %w", err)
|
||||
}
|
||||
if err := s.SetDocumentProcessingStatus(ctx, documentID, tenantID, JobStatusFailed); err != nil && !errors.Is(err, ErrDocumentNotFound) {
|
||||
return false, err
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Exponentielles Backoff: 2^retry_count Sekunden (1s, 2s, 4s, 8s, 16s ...)
|
||||
// ab dem aktuellen Fehlschlag gerechnet.
|
||||
backoff := time.Duration(1<<uint(retryCount)) * time.Second
|
||||
if _, err := s.db.Exec(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'queued', retry_count = $3, error_message = $4,
|
||||
next_attempt_at = now() + ($5::double precision * interval '1 second'), started_at = NULL, updated_at = now()
|
||||
WHERE id = $1 AND tenant_id = $2
|
||||
`, jobID, tenantID, next, jobErr, int64(backoff.Seconds())); err != nil {
|
||||
return false, fmt.Errorf("storage: requeue job: %w", err)
|
||||
}
|
||||
if err := s.SetDocumentProcessingStatus(ctx, documentID, tenantID, JobStatusQueued); err != nil && !errors.Is(err, ErrDocumentNotFound) {
|
||||
return true, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// ReapStaleJobs setzt hängengebliebene 'processing'-Jobs zurück, deren
|
||||
// started_at älter als timeout ist (Prozess-Neustart mitten in der
|
||||
// Verarbeitung, gestorbener OCR-Subprozess, ...). Zurückgesetzt wird wie ein
|
||||
// normaler Fehlschlag: retry_count hoch, exponentielles Backoff, ab
|
||||
// maxRetries dauerhaft 'failed'. Gibt die Anzahl betroffener Jobs zurück.
|
||||
func (s *Store) ReapStaleJobs(ctx context.Context, timeout time.Duration, maxRetries int) (int, error) {
|
||||
secs := int64(timeout.Seconds())
|
||||
if secs < 1 {
|
||||
secs = 1
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, tenant_id, document_id FROM processing_jobs
|
||||
WHERE status = 'processing' AND started_at IS NOT NULL
|
||||
AND started_at < now() - ($1::double precision * interval '1 second')
|
||||
`, secs)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("storage: find stale jobs: %w", err)
|
||||
}
|
||||
type staleJob struct{ id, tenantID, documentID int64 }
|
||||
var stale []staleJob
|
||||
for rows.Next() {
|
||||
var sj staleJob
|
||||
if err := rows.Scan(&sj.id, &sj.tenantID, &sj.documentID); err != nil {
|
||||
rows.Close()
|
||||
return 0, fmt.Errorf("storage: scan stale job: %w", err)
|
||||
}
|
||||
stale = append(stale, sj)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, fmt.Errorf("storage: iterate stale jobs: %w", err)
|
||||
}
|
||||
|
||||
for _, sj := range stale {
|
||||
if _, err := s.MarkJobFailed(ctx, sj.id, sj.tenantID, sj.documentID,
|
||||
fmt.Sprintf("job timed out after %s (reaper)", timeout), maxRetries); err != nil {
|
||||
return len(stale), fmt.Errorf("storage: reap job %d: %w", sj.id, err)
|
||||
}
|
||||
}
|
||||
return len(stale), nil
|
||||
}
|
||||
|
||||
// RequeueJob stellt einen Job (typischerweise einen dauerhaft 'failed'
|
||||
// gelaufenen) wieder in die Queue und setzt retry_count zurück. Wird vom
|
||||
// späteren manuellen Retry-Endpunkt (Phase 3) genutzt; tenant-scoped.
|
||||
func (s *Store) RequeueJob(ctx context.Context, jobID, tenantID int64) error {
|
||||
var documentID int64
|
||||
err := s.db.QueryRow(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'queued', retry_count = 0, error_message = NULL,
|
||||
next_attempt_at = now(), started_at = NULL, updated_at = now()
|
||||
WHERE id = $1 AND tenant_id = $2
|
||||
RETURNING document_id
|
||||
`, jobID, tenantID).Scan(&documentID)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrNoJob
|
||||
}
|
||||
return fmt.Errorf("storage: requeue job: %w", err)
|
||||
}
|
||||
if err := s.SetDocumentProcessingStatus(ctx, documentID, tenantID, JobStatusQueued); err != nil && !errors.Is(err, ErrDocumentNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetJobForDocument liefert den jüngsten Job eines Dokuments (tenant-scoped).
|
||||
// Für den späteren Frontend-Status/Retry-Endpunkt.
|
||||
func (s *Store) GetJobForDocument(ctx context.Context, documentID, tenantID int64) (*ProcessingJob, error) {
|
||||
var j ProcessingJob
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT id, tenant_id, document_id, status, retry_count, derive_title, COALESCE(error_message, ''), next_attempt_at, started_at, created_at, updated_at
|
||||
FROM processing_jobs
|
||||
WHERE document_id = $1 AND tenant_id = $2
|
||||
ORDER BY id DESC LIMIT 1
|
||||
`, documentID, tenantID).Scan(&j.ID, &j.TenantID, &j.DocumentID, &j.Status, &j.RetryCount, &j.DeriveTitle,
|
||||
&j.ErrorMessage, &j.NextAttemptAt, &j.StartedAt, &j.CreatedAt, &j.UpdatedAt)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNoJob
|
||||
}
|
||||
return nil, fmt.Errorf("storage: get job for document: %w", err)
|
||||
}
|
||||
return &j, nil
|
||||
}
|
||||
|
||||
// SetDocumentProcessingStatus aktualisiert documents.processing_status,
|
||||
// tenant-scoped. Reiner UI-/Ablaufstatus (kein ACL-relevanter Inhalt), daher
|
||||
// bewusst OHNE SyncIndex — analog SetDocumentHasThumbnail.
|
||||
func (s *Store) SetDocumentProcessingStatus(ctx context.Context, documentID, tenantID int64, status string) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE documents SET processing_status = $1, updated_at = now() WHERE id = $2 AND tenant_id = $3
|
||||
`, status, documentID, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: set document processing_status: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("storage: set document processing_status: %w", ErrDocumentNotFound)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
// Wiedervorlage (reminder) store. Pattern ported from archivmail's
|
||||
// saved_searches.go: a small, single-file store extending the shared *Store
|
||||
// with its own idempotent schema init + CRUD, ownership enforced by
|
||||
// requiring id+tenant_id(+user_id) to match on every mutating query.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// Reminder ("Wiedervorlage") is a due-date follow-up attached to a document.
|
||||
type Reminder struct {
|
||||
ID int64 `json:"id"`
|
||||
DocumentID int64 `json:"document_id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
UserID int64 `json:"user_id"`
|
||||
DueDate time.Time `json:"due_date"`
|
||||
Note string `json:"note,omitempty"`
|
||||
Status string `json:"status"` // open|done|dismissed
|
||||
NotifiedAt *time.Time `json:"notified_at,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
const (
|
||||
ReminderStatusOpen = "open"
|
||||
ReminderStatusDone = "done"
|
||||
ReminderStatusDismissed = "dismissed"
|
||||
)
|
||||
|
||||
func (s *Store) initReminderSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS reminders (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id),
|
||||
tenant_id BIGINT NOT NULL,
|
||||
user_id BIGINT NOT NULL,
|
||||
due_date TIMESTAMPTZ NOT NULL,
|
||||
note TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'open',
|
||||
notified_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_reminders_tenant_due ON reminders(tenant_id, due_date) WHERE status = 'open';
|
||||
CREATE INDEX IF NOT EXISTS idx_reminders_document ON reminders(document_id);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create reminders table: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateReminder inserts a new reminder for a document and returns it.
|
||||
func (s *Store) CreateReminder(ctx context.Context, documentID, tenantID, userID int64, dueDate time.Time, note string) (*Reminder, error) {
|
||||
var r Reminder
|
||||
err := s.db.QueryRow(ctx, `
|
||||
INSERT INTO reminders (document_id, tenant_id, user_id, due_date, note, status)
|
||||
VALUES ($1, $2, $3, $4, $5, 'open')
|
||||
RETURNING id, document_id, tenant_id, user_id, due_date, COALESCE(note, ''), status, notified_at, created_at, updated_at
|
||||
`, documentID, tenantID, userID, dueDate, nullIfEmpty(note),
|
||||
).Scan(&r.ID, &r.DocumentID, &r.TenantID, &r.UserID, &r.DueDate, &r.Note, &r.Status, &r.NotifiedAt, &r.CreatedAt, &r.UpdatedAt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reminders: create: %w", err)
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// ListReminders returns reminders for a user within a tenant, optionally
|
||||
// filtered by status ("" = all).
|
||||
func (s *Store) ListReminders(ctx context.Context, tenantID, userID int64, status string) ([]Reminder, error) {
|
||||
var rows pgx.Rows
|
||||
var err error
|
||||
|
||||
if status == "" {
|
||||
rows, err = s.db.Query(ctx, `
|
||||
SELECT id, document_id, tenant_id, user_id, due_date, COALESCE(note, ''), status, notified_at, created_at, updated_at
|
||||
FROM reminders WHERE tenant_id = $1 AND user_id = $2 ORDER BY due_date ASC
|
||||
`, tenantID, userID)
|
||||
} else {
|
||||
rows, err = s.db.Query(ctx, `
|
||||
SELECT id, document_id, tenant_id, user_id, due_date, COALESCE(note, ''), status, notified_at, created_at, updated_at
|
||||
FROM reminders WHERE tenant_id = $1 AND user_id = $2 AND status = $3 ORDER BY due_date ASC
|
||||
`, tenantID, userID, status)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reminders: list: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]Reminder, 0)
|
||||
for rows.Next() {
|
||||
var r Reminder
|
||||
if err := rows.Scan(&r.ID, &r.DocumentID, &r.TenantID, &r.UserID, &r.DueDate, &r.Note, &r.Status, &r.NotifiedAt, &r.CreatedAt, &r.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("reminders: scan: %w", err)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListDueReminders returns all open reminders with due_date <= cutoff and
|
||||
// notified_at still NULL, across all tenants. Intended for the cron
|
||||
// notification job (`archivdms reminders notify`).
|
||||
func (s *Store) ListDueReminders(ctx context.Context, cutoff time.Time) ([]Reminder, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, document_id, tenant_id, user_id, due_date, COALESCE(note, ''), status, notified_at, created_at, updated_at
|
||||
FROM reminders
|
||||
WHERE status = 'open' AND due_date <= $1 AND notified_at IS NULL
|
||||
ORDER BY due_date ASC
|
||||
`, cutoff)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reminders: list due: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []Reminder
|
||||
for rows.Next() {
|
||||
var r Reminder
|
||||
if err := rows.Scan(&r.ID, &r.DocumentID, &r.TenantID, &r.UserID, &r.DueDate, &r.Note, &r.Status, &r.NotifiedAt, &r.CreatedAt, &r.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("reminders: scan due: %w", err)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// UpdateReminderStatus updates the status of a reminder, enforcing ownership
|
||||
// via id+tenant_id+user_id.
|
||||
func (s *Store) UpdateReminderStatus(ctx context.Context, id, tenantID, userID int64, status string) (*Reminder, error) {
|
||||
var r Reminder
|
||||
err := s.db.QueryRow(ctx, `
|
||||
UPDATE reminders SET status = $1, updated_at = now()
|
||||
WHERE id = $2 AND tenant_id = $3 AND user_id = $4
|
||||
RETURNING id, document_id, tenant_id, user_id, due_date, COALESCE(note, ''), status, notified_at, created_at, updated_at
|
||||
`, status, id, tenantID, userID,
|
||||
).Scan(&r.ID, &r.DocumentID, &r.TenantID, &r.UserID, &r.DueDate, &r.Note, &r.Status, &r.NotifiedAt, &r.CreatedAt, &r.UpdatedAt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reminders: update status: %w", err)
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// MarkReminderNotified sets notified_at = now() for the given reminder. Used
|
||||
// by the cron notification job after a successful send; not ownership-scoped
|
||||
// because it runs as a system job, not on behalf of a specific user.
|
||||
func (s *Store) MarkReminderNotified(ctx context.Context, id int64) error {
|
||||
_, err := s.db.Exec(ctx, `UPDATE reminders SET notified_at = now(), updated_at = now() WHERE id = $1`, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reminders: mark notified: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteReminder deletes a reminder, enforcing ownership via
|
||||
// id+tenant_id+user_id.
|
||||
func (s *Store) DeleteReminder(ctx context.Context, id, tenantID, userID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `DELETE FROM reminders WHERE id = $1 AND tenant_id = $2 AND user_id = $3`, id, tenantID, userID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reminders: delete: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("reminders: not found or not owned by user")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,533 @@
|
||||
// GoBD-Aufbewahrungsregeln (retention rules / "Disposition Schedules").
|
||||
//
|
||||
// Naming/model reference: Alfresco's Disposition-Schedule terminology
|
||||
// (trigger -> retention period -> disposition), but NOT its architecture —
|
||||
// archivdms stays single-binary/Postgres. A retention rule declares, per
|
||||
// document type (or tenant-wide as a default with doc_type_id IS NULL), HOW
|
||||
// LONG a document must be kept and from WHICH base date the retention period
|
||||
// starts counting. The batch job ApplyRetentionRules computes and SETS
|
||||
// documents.retain_until from the matching rule — it never hard-deletes and
|
||||
// never shortens an existing lock. Actual disposition (final deletion) still
|
||||
// runs through the existing Papierkorb + Vier-Augen delete-request flow in
|
||||
// trash.go, which re-checks retain_until before allowing deletion.
|
||||
//
|
||||
// State model (deliberately reusing existing fields, no new status table):
|
||||
// - "locked" : documents.retain_until IS NOT NULL AND >= now()
|
||||
// (WORM lock, blocks CreateDeleteRequest/Confirm)
|
||||
// - "eligible_for_disposition": retain_until IS NOT NULL AND < now() AND
|
||||
// deleted_at IS NULL (see ListEligibleForDisposition)
|
||||
// - disposition proper : handled by trash.go once a human soft-deletes
|
||||
// the document and starts the Vier-Augen request.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// ErrRetentionRuleNotFound is returned when a tenant-scoped retention-rule
|
||||
// lookup/update/delete affects zero rows (wrong id or wrong tenant).
|
||||
var ErrRetentionRuleNotFound = errors.New("storage: retention rule not found for tenant")
|
||||
|
||||
// Trigger types: how the retention base date is derived for a document.
|
||||
const (
|
||||
// RetentionTriggerDocumentDate uses documents.document_date (belegdatum),
|
||||
// falling back to created_at when no document_date is set.
|
||||
RetentionTriggerDocumentDate = "document_date"
|
||||
// RetentionTriggerUploadDate uses documents.created_at (scan/upload time).
|
||||
RetentionTriggerUploadDate = "upload_date"
|
||||
// RetentionTriggerFixedDate uses trigger_reference parsed as 2006-01-02 as
|
||||
// the base date for every matching document (one-time legal cutoff).
|
||||
RetentionTriggerFixedDate = "fixed_date"
|
||||
// RetentionTriggerEvent marks event-based retention (e.g. Geschäftsjahres-
|
||||
// ende / Vertragsende). NOT auto-computed by ApplyRetentionRules — the
|
||||
// system does not yet observe such events. Documents matched by an event
|
||||
// rule are skipped (retain_until left NULL). This is the known future
|
||||
// extension point.
|
||||
RetentionTriggerEvent = "event"
|
||||
)
|
||||
|
||||
// RetentionRule is a per-tenant GoBD retention policy. doc_type_id NULL makes
|
||||
// it the tenant-wide default rule (lowest precedence). A UNIQUE(tenant_id,
|
||||
// doc_type_id) constraint guarantees at most one active rule per (tenant,
|
||||
// doc_type), so no "strictest wins" tie-break logic is needed.
|
||||
type RetentionRule struct {
|
||||
ID int64 `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
DocTypeID *int64 `json:"doc_type_id,omitempty"`
|
||||
Name string `json:"name"`
|
||||
TriggerType string `json:"trigger_type"`
|
||||
TriggerReference string `json:"trigger_reference,omitempty"`
|
||||
RetentionYears *int `json:"retention_years,omitempty"`
|
||||
RetentionDays *int `json:"retention_days,omitempty"`
|
||||
LegalBasis string `json:"legal_basis,omitempty"`
|
||||
RequiresApprovalForDestroy bool `json:"requires_approval_for_destroy"`
|
||||
DSGVOConflict bool `json:"dsgvo_conflict"`
|
||||
Active bool `json:"active"`
|
||||
CreatedBy *int64 `json:"created_by,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// RetentionPreview is one line of a dry-run: which retain_until WOULD be set
|
||||
// on which document by which rule, without writing anything.
|
||||
type RetentionPreview struct {
|
||||
DocumentID int64 `json:"document_id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
RuleID int64 `json:"rule_id"`
|
||||
RuleName string `json:"rule_name"`
|
||||
RetainUntil *time.Time `json:"retain_until"`
|
||||
}
|
||||
|
||||
func (s *Store) initRetentionRulesSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS retention_rules (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
doc_type_id BIGINT REFERENCES document_types(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
trigger_type TEXT NOT NULL
|
||||
CHECK (trigger_type IN ('document_date','upload_date','fixed_date','event')),
|
||||
trigger_reference TEXT NOT NULL DEFAULT '',
|
||||
retention_years INT,
|
||||
retention_days INT,
|
||||
legal_basis TEXT NOT NULL DEFAULT '',
|
||||
requires_approval_for_destroy BOOLEAN NOT NULL DEFAULT true,
|
||||
dsgvo_conflict BOOLEAN NOT NULL DEFAULT false,
|
||||
active BOOLEAN NOT NULL DEFAULT true,
|
||||
created_by BIGINT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, doc_type_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_retention_rules_tenant ON retention_rules(tenant_id) WHERE active;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create retention_rules table: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const retentionRuleCols = `id, tenant_id, doc_type_id, name, trigger_type, trigger_reference,
|
||||
retention_years, retention_days, legal_basis, requires_approval_for_destroy,
|
||||
dsgvo_conflict, active, created_by, created_at, updated_at`
|
||||
|
||||
func scanRetentionRule(row pgx.Row, r *RetentionRule) error {
|
||||
return row.Scan(&r.ID, &r.TenantID, &r.DocTypeID, &r.Name, &r.TriggerType, &r.TriggerReference,
|
||||
&r.RetentionYears, &r.RetentionDays, &r.LegalBasis, &r.RequiresApprovalForDestroy,
|
||||
&r.DSGVOConflict, &r.Active, &r.CreatedBy, &r.CreatedAt, &r.UpdatedAt)
|
||||
}
|
||||
|
||||
// validateRetentionRule enforces the cross-field invariants that a CHECK
|
||||
// constraint cannot express: non-event rules must carry at least one of
|
||||
// retention_years/retention_days (otherwise retain_until would equal the base
|
||||
// date, a misconfiguration), and fixed_date rules must carry a parseable
|
||||
// trigger_reference date.
|
||||
func validateRetentionRule(r RetentionRule) error {
|
||||
if r.Name == "" {
|
||||
return fmt.Errorf("retention rule: name is required")
|
||||
}
|
||||
switch r.TriggerType {
|
||||
case RetentionTriggerDocumentDate, RetentionTriggerUploadDate, RetentionTriggerFixedDate:
|
||||
yrs := 0
|
||||
if r.RetentionYears != nil {
|
||||
yrs = *r.RetentionYears
|
||||
}
|
||||
days := 0
|
||||
if r.RetentionDays != nil {
|
||||
days = *r.RetentionDays
|
||||
}
|
||||
if yrs <= 0 && days <= 0 {
|
||||
return fmt.Errorf("retention rule: trigger_type %q requires retention_years and/or retention_days", r.TriggerType)
|
||||
}
|
||||
if r.TriggerType == RetentionTriggerFixedDate {
|
||||
if _, err := time.Parse("2006-01-02", r.TriggerReference); err != nil {
|
||||
return fmt.Errorf("retention rule: fixed_date trigger_reference must be YYYY-MM-DD: %w", err)
|
||||
}
|
||||
}
|
||||
case RetentionTriggerEvent:
|
||||
// event rules never auto-compute retain_until; retention_years/days are
|
||||
// optional and only informational until the event-observation feature
|
||||
// exists.
|
||||
default:
|
||||
return fmt.Errorf("retention rule: invalid trigger_type %q", r.TriggerType)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateRetentionRule inserts a new retention rule for a tenant and returns it.
|
||||
func (s *Store) CreateRetentionRule(ctx context.Context, tenantID int64, rule RetentionRule) (*RetentionRule, error) {
|
||||
if err := validateRetentionRule(rule); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var r RetentionRule
|
||||
err := scanRetentionRule(s.db.QueryRow(ctx, `
|
||||
INSERT INTO retention_rules
|
||||
(tenant_id, doc_type_id, name, trigger_type, trigger_reference,
|
||||
retention_years, retention_days, legal_basis, requires_approval_for_destroy,
|
||||
dsgvo_conflict, active, created_by)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
||||
RETURNING `+retentionRuleCols,
|
||||
tenantID, rule.DocTypeID, rule.Name, rule.TriggerType, rule.TriggerReference,
|
||||
rule.RetentionYears, rule.RetentionDays, rule.LegalBasis, rule.RequiresApprovalForDestroy,
|
||||
rule.DSGVOConflict, rule.Active, rule.CreatedBy), &r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("retention rules: create: %w", err)
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// ListRetentionRules returns all retention rules for a tenant, tenant-wide
|
||||
// default (doc_type_id IS NULL) last.
|
||||
func (s *Store) ListRetentionRules(ctx context.Context, tenantID int64) ([]RetentionRule, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT `+retentionRuleCols+`
|
||||
FROM retention_rules WHERE tenant_id = $1
|
||||
ORDER BY doc_type_id IS NULL, doc_type_id, id
|
||||
`, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("retention rules: list: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]RetentionRule, 0)
|
||||
for rows.Next() {
|
||||
var r RetentionRule
|
||||
if err := scanRetentionRule(rows, &r); err != nil {
|
||||
return nil, fmt.Errorf("retention rules: scan: %w", err)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// GetRetentionRule returns a single retention rule, tenant-scoped.
|
||||
func (s *Store) GetRetentionRule(ctx context.Context, id, tenantID int64) (*RetentionRule, error) {
|
||||
var r RetentionRule
|
||||
err := scanRetentionRule(s.db.QueryRow(ctx, `
|
||||
SELECT `+retentionRuleCols+`
|
||||
FROM retention_rules WHERE id = $1 AND tenant_id = $2
|
||||
`, id, tenantID), &r)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrRetentionRuleNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("retention rules: get: %w", err)
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// UpdateRetentionRule updates a retention rule in place, tenant-scoped.
|
||||
func (s *Store) UpdateRetentionRule(ctx context.Context, id, tenantID int64, rule RetentionRule) (*RetentionRule, error) {
|
||||
if err := validateRetentionRule(rule); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var r RetentionRule
|
||||
err := scanRetentionRule(s.db.QueryRow(ctx, `
|
||||
UPDATE retention_rules SET
|
||||
doc_type_id = $3,
|
||||
name = $4,
|
||||
trigger_type = $5,
|
||||
trigger_reference = $6,
|
||||
retention_years = $7,
|
||||
retention_days = $8,
|
||||
legal_basis = $9,
|
||||
requires_approval_for_destroy = $10,
|
||||
dsgvo_conflict = $11,
|
||||
active = $12,
|
||||
updated_at = now()
|
||||
WHERE id = $1 AND tenant_id = $2
|
||||
RETURNING `+retentionRuleCols,
|
||||
id, tenantID, rule.DocTypeID, rule.Name, rule.TriggerType, rule.TriggerReference,
|
||||
rule.RetentionYears, rule.RetentionDays, rule.LegalBasis, rule.RequiresApprovalForDestroy,
|
||||
rule.DSGVOConflict, rule.Active), &r)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrRetentionRuleNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("retention rules: update: %w", err)
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// DeleteRetentionRule removes a retention rule, tenant-scoped. Deleting a rule
|
||||
// does NOT retroactively clear retain_until on documents it previously locked
|
||||
// (GoBD: a WORM lock, once set, is never shortened).
|
||||
func (s *Store) DeleteRetentionRule(ctx context.Context, id, tenantID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `DELETE FROM retention_rules WHERE id = $1 AND tenant_id = $2`, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("retention rules: delete: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrRetentionRuleNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// computeRuleRetainUntil is the pure retain_until computation shared by the batch
|
||||
// job and the dry-run preview. It returns (retainUntil, ok): ok=false means
|
||||
// the document must be SKIPPED (event-based rule, or a misconfigured
|
||||
// years+days==0 rule that validation should have prevented but we defend
|
||||
// against anyway). It never writes to the DB.
|
||||
//
|
||||
// Base date by trigger_type:
|
||||
// - document_date: doc.DocumentDate if set, else doc.CreatedAt
|
||||
// - upload_date: doc.CreatedAt
|
||||
// - fixed_date: rule.TriggerReference parsed as 2006-01-02
|
||||
// - event: skipped (ok=false)
|
||||
//
|
||||
// retain_until = base + retention_years years + retention_days days.
|
||||
func computeRuleRetainUntil(rule RetentionRule, doc Document) (*time.Time, bool) {
|
||||
var base time.Time
|
||||
switch rule.TriggerType {
|
||||
case RetentionTriggerDocumentDate:
|
||||
if doc.DocumentDate != nil {
|
||||
base = *doc.DocumentDate
|
||||
} else {
|
||||
base = doc.CreatedAt
|
||||
}
|
||||
case RetentionTriggerUploadDate:
|
||||
base = doc.CreatedAt
|
||||
case RetentionTriggerFixedDate:
|
||||
parsed, err := time.Parse("2006-01-02", rule.TriggerReference)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
base = parsed
|
||||
case RetentionTriggerEvent:
|
||||
// Not auto-computed — requires a real trigger event the system does not
|
||||
// yet observe. Future extension point.
|
||||
return nil, false
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
|
||||
years, days := 0, 0
|
||||
if rule.RetentionYears != nil {
|
||||
years = *rule.RetentionYears
|
||||
}
|
||||
if rule.RetentionDays != nil {
|
||||
days = *rule.RetentionDays
|
||||
}
|
||||
if years <= 0 && days <= 0 {
|
||||
// Misconfigured rule — never produce retain_until == base silently.
|
||||
return nil, false
|
||||
}
|
||||
|
||||
ru := base.AddDate(years, 0, days)
|
||||
return &ru, true
|
||||
}
|
||||
|
||||
// tenantIDsWithRules returns the distinct tenant_ids that have at least one
|
||||
// active retention rule. Used by ApplyRetentionRules/PreviewRetentionRules when
|
||||
// tenantID == 0 ("all tenants").
|
||||
func (s *Store) tenantIDsWithRules(ctx context.Context) ([]int64, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT DISTINCT tenant_id FROM retention_rules WHERE active`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("retention rules: distinct tenants: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("retention rules: scan tenant: %w", err)
|
||||
}
|
||||
out = append(out, id)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// matchDocumentsForRule loads documents of a tenant that (a) have no
|
||||
// retain_until yet, (b) are not in the trash, and (c) match the given rule's
|
||||
// scope: for a doc-type-specific rule (doc_type_id NOT NULL) exactly that
|
||||
// doc_type; for the tenant-wide default rule (doc_type_id IS NULL) every
|
||||
// document whose doc_type_id has NO own specific active rule (so the specific
|
||||
// rule always wins the precedence). Only the columns computeRuleRetainUntil needs
|
||||
// are loaded.
|
||||
func (s *Store) matchDocumentsForRule(ctx context.Context, tenantID int64, rule RetentionRule) ([]Document, error) {
|
||||
var (
|
||||
rows pgx.Rows
|
||||
err error
|
||||
)
|
||||
if rule.DocTypeID != nil {
|
||||
rows, err = s.db.Query(ctx, `
|
||||
SELECT id, tenant_id, document_date, created_at
|
||||
FROM documents
|
||||
WHERE tenant_id = $1 AND deleted_at IS NULL
|
||||
AND retain_until IS NULL
|
||||
AND doc_type_id = $2
|
||||
`, tenantID, *rule.DocTypeID)
|
||||
} else {
|
||||
// Tenant-wide default: only documents whose doc_type_id has no own
|
||||
// active specific rule (NULL doc_type_id included). Specific rule wins.
|
||||
rows, err = s.db.Query(ctx, `
|
||||
SELECT d.id, d.tenant_id, d.document_date, d.created_at
|
||||
FROM documents d
|
||||
WHERE d.tenant_id = $1 AND d.deleted_at IS NULL
|
||||
AND d.retain_until IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM retention_rules r
|
||||
WHERE r.tenant_id = $1 AND r.active
|
||||
AND r.doc_type_id IS NOT NULL
|
||||
AND r.doc_type_id = d.doc_type_id
|
||||
)
|
||||
`, tenantID)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("retention rules: match documents: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []Document
|
||||
for rows.Next() {
|
||||
var d Document
|
||||
if err := rows.Scan(&d.ID, &d.TenantID, &d.DocumentDate, &d.CreatedAt); err != nil {
|
||||
return nil, fmt.Errorf("retention rules: scan matched document: %w", err)
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// PreviewRetentionRules computes (without writing) which retain_until values
|
||||
// ApplyRetentionRules WOULD set. tenantID == 0 means all tenants that have
|
||||
// active rules. Documents skipped by computeRuleRetainUntil (event rules,
|
||||
// misconfigured rules) are omitted from the preview.
|
||||
func (s *Store) PreviewRetentionRules(ctx context.Context, tenantID int64) ([]RetentionPreview, error) {
|
||||
tenants := []int64{tenantID}
|
||||
if tenantID == 0 {
|
||||
var err error
|
||||
tenants, err = s.tenantIDsWithRules(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var out []RetentionPreview
|
||||
for _, tid := range tenants {
|
||||
rules, err := s.ListRetentionRules(ctx, tid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, rule := range rules {
|
||||
if !rule.Active {
|
||||
continue
|
||||
}
|
||||
docs, err := s.matchDocumentsForRule(ctx, tid, rule)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, doc := range docs {
|
||||
ru, ok := computeRuleRetainUntil(rule, doc)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, RetentionPreview{
|
||||
DocumentID: doc.ID,
|
||||
TenantID: tid,
|
||||
RuleID: rule.ID,
|
||||
RuleName: rule.Name,
|
||||
RetainUntil: ru,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ApplyRetentionRules scans documents for the tenant (or all tenants if
|
||||
// tenantID == 0) whose retain_until is NULL and which have a matching active
|
||||
// retention rule (by doc_type_id, falling back to the tenant-wide default rule
|
||||
// with doc_type_id IS NULL), computes retain_until from
|
||||
// trigger_type/trigger_reference + retention_years/retention_days, and sets it.
|
||||
//
|
||||
// Never touches documents that already have retain_until set (a rule change
|
||||
// does not retroactively shrink an existing lock — GoBD: once WORM, only ever
|
||||
// extend, never shorten; extension is a separate future feature, not
|
||||
// implemented here). Event-based rules are skipped (see computeRuleRetainUntil).
|
||||
// Returns the number of documents updated.
|
||||
func (s *Store) ApplyRetentionRules(ctx context.Context, tenantID int64) (int, error) {
|
||||
tenants := []int64{tenantID}
|
||||
if tenantID == 0 {
|
||||
var err error
|
||||
tenants, err = s.tenantIDsWithRules(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
updated := 0
|
||||
for _, tid := range tenants {
|
||||
rules, err := s.ListRetentionRules(ctx, tid)
|
||||
if err != nil {
|
||||
return updated, err
|
||||
}
|
||||
for _, rule := range rules {
|
||||
if !rule.Active {
|
||||
continue
|
||||
}
|
||||
docs, err := s.matchDocumentsForRule(ctx, tid, rule)
|
||||
if err != nil {
|
||||
return updated, err
|
||||
}
|
||||
for _, doc := range docs {
|
||||
ru, ok := computeRuleRetainUntil(rule, doc)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
// Re-assert retain_until IS NULL in the WHERE so a concurrent
|
||||
// run / manual set is never overwritten (never shorten a lock).
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE documents SET retain_until = $3, updated_at = now()
|
||||
WHERE id = $1 AND tenant_id = $2 AND retain_until IS NULL AND deleted_at IS NULL
|
||||
`, doc.ID, tid, *ru)
|
||||
if err != nil {
|
||||
return updated, fmt.Errorf("retention rules: set retain_until: %w", err)
|
||||
}
|
||||
updated += int(tag.RowsAffected())
|
||||
}
|
||||
}
|
||||
}
|
||||
return updated, nil
|
||||
}
|
||||
|
||||
// ListEligibleForDisposition returns documents whose retention has expired
|
||||
// (retain_until IS NOT NULL AND < now()) but which are not yet in the trash
|
||||
// (deleted_at IS NULL). These are implicitly "eligible for disposition": a
|
||||
// human can now soft-delete them and start the Vier-Augen delete-request flow
|
||||
// (trash.go). No new status is invented — eligibility is derived from
|
||||
// retain_until alone.
|
||||
func (s *Store) ListEligibleForDisposition(ctx context.Context, tenantID int64) ([]Document, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, tenant_id, title, COALESCE(doc_type, ''), COALESCE(correspondent, ''),
|
||||
doc_type_id, correspondent_id, storage_path, content_hash, COALESCE(ocr_text, ''),
|
||||
retain_until, document_date, COALESCE(source, ''), COALESCE(source_ref, ''),
|
||||
created_by, title_manually_set, has_thumbnail, created_at, updated_at
|
||||
FROM documents
|
||||
WHERE tenant_id = $1 AND deleted_at IS NULL
|
||||
AND retain_until IS NOT NULL AND retain_until < now()
|
||||
ORDER BY retain_until ASC
|
||||
`, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("retention rules: list eligible for disposition: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]Document, 0)
|
||||
for rows.Next() {
|
||||
var d Document
|
||||
if err := rows.Scan(&d.ID, &d.TenantID, &d.Title, &d.DocType, &d.Correspondent,
|
||||
&d.DocTypeID, &d.CorrespondentID, &d.StoragePath, &d.ContentHash, &d.OCRText,
|
||||
&d.RetainUntil, &d.DocumentDate, &d.Source, &d.SourceRef,
|
||||
&d.CreatedBy, &d.TitleManuallySet, &d.HasThumbnail, &d.CreatedAt, &d.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("retention rules: scan eligible document: %w", err)
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// ErrSavedViewNotFound is returned when a saved-view lookup/update/delete
|
||||
// affects zero rows (wrong id, wrong tenant, or — for mutations — the caller
|
||||
// is not the view's creator).
|
||||
var ErrSavedViewNotFound = errors.New("storage: saved view not found")
|
||||
|
||||
// ErrSavedViewForbidden is returned by UpdateSavedView/DeleteSavedView when the
|
||||
// view exists within the tenant but the requester is not its creator (only the
|
||||
// creator may modify a shared view).
|
||||
var ErrSavedViewForbidden = errors.New("storage: not allowed to modify this saved view")
|
||||
|
||||
// SavedView is a named, reusable search/filter query (Paperless-ngx inspired).
|
||||
// Filters stores the serialized search query verbatim (JSONB) so the client can
|
||||
// re-hydrate it 1:1 into a new search request. A view is private to its creator
|
||||
// unless IsShared is set, in which case it is visible to every user of the
|
||||
// tenant — but still only editable/deletable by its creator.
|
||||
type SavedView struct {
|
||||
ID int64 `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
UserID int64 `json:"user_id"`
|
||||
Name string `json:"name"`
|
||||
Filters json.RawMessage `json:"filters"`
|
||||
IsShared bool `json:"is_shared"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// initSavedViewsSchema creates the saved_views table. Idempotent; wired into
|
||||
// Store.initSchema after initDocumentNotesSchema (see documents.go).
|
||||
func (s *Store) initSavedViewsSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS saved_views (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
user_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
filters JSONB NOT NULL,
|
||||
is_shared BOOLEAN NOT NULL DEFAULT false,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_saved_views_tenant_user ON saved_views (tenant_id, user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_saved_views_tenant_shared ON saved_views (tenant_id, is_shared) WHERE is_shared;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create saved_views table: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateSavedView inserts a new saved view for the given user/tenant and
|
||||
// returns it.
|
||||
func (s *Store) CreateSavedView(ctx context.Context, tenantID, userID int64, name string, filters json.RawMessage, isShared bool) (*SavedView, error) {
|
||||
var v SavedView
|
||||
err := s.db.QueryRow(ctx, `
|
||||
INSERT INTO saved_views (tenant_id, user_id, name, filters, is_shared)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING id, tenant_id, user_id, name, filters, is_shared, created_at, updated_at
|
||||
`, tenantID, userID, name, filters, isShared).Scan(
|
||||
&v.ID, &v.TenantID, &v.UserID, &v.Name, &v.Filters, &v.IsShared, &v.CreatedAt, &v.UpdatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: create saved view: %w", err)
|
||||
}
|
||||
return &v, nil
|
||||
}
|
||||
|
||||
// ListSavedViews returns all views the user may see within the tenant: their
|
||||
// own views plus every view shared tenant-wide (is_shared). Ordered by name.
|
||||
func (s *Store) ListSavedViews(ctx context.Context, tenantID, userID int64) ([]SavedView, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, tenant_id, user_id, name, filters, is_shared, created_at, updated_at
|
||||
FROM saved_views
|
||||
WHERE tenant_id = $1 AND (user_id = $2 OR is_shared)
|
||||
ORDER BY name ASC, id ASC
|
||||
`, tenantID, userID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list saved views: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]SavedView, 0)
|
||||
for rows.Next() {
|
||||
var v SavedView
|
||||
if err := rows.Scan(&v.ID, &v.TenantID, &v.UserID, &v.Name, &v.Filters, &v.IsShared, &v.CreatedAt, &v.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan saved view: %w", err)
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// UpdateSavedView updates a view's name/filters/is_shared. Only the creator may
|
||||
// update it: the WHERE clause matches id + tenant_id + user_id. If the view does
|
||||
// not exist within the tenant, ErrSavedViewNotFound is returned; if it exists
|
||||
// but belongs to another user, ErrSavedViewForbidden is returned (so the caller
|
||||
// can distinguish 404 from 403).
|
||||
func (s *Store) UpdateSavedView(ctx context.Context, id, tenantID, userID int64, name string, filters json.RawMessage, isShared bool) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE saved_views
|
||||
SET name = $4, filters = $5, is_shared = $6, updated_at = now()
|
||||
WHERE id = $1 AND tenant_id = $2 AND user_id = $3
|
||||
`, id, tenantID, userID, name, filters, isShared)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: update saved view: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return s.savedViewMissReason(ctx, id, tenantID, userID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteSavedView deletes a view. Only the creator may delete it (WHERE
|
||||
// id + tenant_id + user_id). Returns ErrSavedViewNotFound / ErrSavedViewForbidden
|
||||
// analogous to UpdateSavedView.
|
||||
func (s *Store) DeleteSavedView(ctx context.Context, id, tenantID, userID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
DELETE FROM saved_views WHERE id = $1 AND tenant_id = $2 AND user_id = $3
|
||||
`, id, tenantID, userID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: delete saved view: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return s.savedViewMissReason(ctx, id, tenantID, userID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// savedViewMissReason disambiguates a zero-rows mutation: the view either does
|
||||
// not exist for the tenant at all (ErrSavedViewNotFound) or it exists but is
|
||||
// owned by another user (ErrSavedViewForbidden).
|
||||
func (s *Store) savedViewMissReason(ctx context.Context, id, tenantID, userID int64) error {
|
||||
var ownerID int64
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT user_id FROM saved_views WHERE id = $1 AND tenant_id = $2
|
||||
`, id, tenantID).Scan(&ownerID)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrSavedViewNotFound
|
||||
}
|
||||
return fmt.Errorf("storage: lookup saved view owner: %w", err)
|
||||
}
|
||||
if ownerID != userID {
|
||||
return ErrSavedViewForbidden
|
||||
}
|
||||
return ErrSavedViewNotFound
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"archivdms/internal/index"
|
||||
)
|
||||
|
||||
// SearchResultDoc is a single search hit: the full document row (re-hydrated
|
||||
// from Postgres, the source of truth) plus its BM25 relevance score from the
|
||||
// Manticore index. Document is embedded so the JSON shape stays identical to
|
||||
// the /api/documents list response, with an added top-level "score" field.
|
||||
type SearchResultDoc struct {
|
||||
Document
|
||||
Score float64 `json:"score"`
|
||||
}
|
||||
|
||||
// SearchResult is the paginated envelope returned by SearchDocuments.
|
||||
type SearchResult struct {
|
||||
Results []SearchResultDoc `json:"results"`
|
||||
Total int `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
}
|
||||
|
||||
// ErrSearchUnavailable is returned by SearchDocuments when no search index is
|
||||
// wired into the store (empty Manticore DSN). The handler maps this to a 503 —
|
||||
// unlike the best-effort write-path sync helpers, a search request must fail
|
||||
// loudly rather than silently return an empty result.
|
||||
var ErrSearchUnavailable = ErrNoIndexer
|
||||
|
||||
// SearchDocuments runs a full-text + attribute query against the tenant's
|
||||
// Manticore index, then re-hydrates the matching document rows from Postgres
|
||||
// (authoritative), preserving the index's relevance ordering and attaching each
|
||||
// hit's score.
|
||||
//
|
||||
// The index only ever returns documents.id + score; the WHERE tenant_id / and
|
||||
// deleted_at IS NULL clause below is the authoritative ownership + soft-delete
|
||||
// boundary — the index is treated as a hint, never as the source of truth.
|
||||
// Returns ErrSearchUnavailable when no indexer is configured.
|
||||
func (s *Store) SearchDocuments(ctx context.Context, tenantID int64, q index.SearchQuery) (*SearchResult, error) {
|
||||
if s.indexer == nil {
|
||||
return nil, ErrSearchUnavailable
|
||||
}
|
||||
|
||||
page := q.Page
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
pageSize := q.PageSize
|
||||
if pageSize <= 0 {
|
||||
pageSize = 20
|
||||
}
|
||||
q.Page = page
|
||||
q.PageSize = pageSize
|
||||
|
||||
hits, total, err := s.indexer.ForTenant(tenantID).Search(ctx, q)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: search documents: %w", err)
|
||||
}
|
||||
|
||||
result := &SearchResult{
|
||||
Results: []SearchResultDoc{},
|
||||
Total: total,
|
||||
Page: page,
|
||||
PageSize: pageSize,
|
||||
}
|
||||
if len(hits) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Collect the hit ids (preserving score order) for a single batch SELECT.
|
||||
ids := make([]int64, len(hits))
|
||||
scoreByID := make(map[int64]float64, len(hits))
|
||||
for i, h := range hits {
|
||||
ids[i] = h.ID
|
||||
scoreByID[h.ID] = h.Score
|
||||
}
|
||||
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, tenant_id, title, COALESCE(doc_type, ''), COALESCE(correspondent, ''), doc_type_id, correspondent_id, storage_path, content_hash,
|
||||
COALESCE(ocr_text, ''), retain_until, COALESCE(source, ''), COALESCE(source_ref, ''), created_at, updated_at
|
||||
FROM documents
|
||||
WHERE id = ANY($1) AND tenant_id = $2 AND deleted_at IS NULL
|
||||
`, ids, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: search hydrate documents: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
docByID := make(map[int64]Document, len(hits))
|
||||
for rows.Next() {
|
||||
var d Document
|
||||
if err := rows.Scan(&d.ID, &d.TenantID, &d.Title, &d.DocType, &d.Correspondent, &d.DocTypeID, &d.CorrespondentID, &d.StoragePath, &d.ContentHash,
|
||||
&d.OCRText, &d.RetainUntil, &d.Source, &d.SourceRef, &d.CreatedAt, &d.UpdatedAt); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan search document: %w", err)
|
||||
}
|
||||
docByID[d.ID] = d
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("storage: search rows: %w", err)
|
||||
}
|
||||
|
||||
// Emit in the index's ranking order; skip ids the DB dropped (a stale index
|
||||
// entry for a since-deleted / re-tenanted document).
|
||||
for _, id := range ids {
|
||||
d, ok := docByID[id]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
result.Results = append(result.Results, SearchResultDoc{Document: d, Score: scoreByID[id]})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
// SFTP credential store. Pattern ported from internal/userstore.go
|
||||
// (bcrypt-hashed secret, timing-safe "not found" handling) but deliberately
|
||||
// kept separate from the `users` table: an SFTP credential is a narrow,
|
||||
// independently revocable secret scoped to exactly one tenant's inbox
|
||||
// folder, not a full login account. See internal/sftpserver for the server
|
||||
// that authenticates against this store and internal/api/sftp_handlers.go
|
||||
// for the admin CRUD API.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
const sftpBcryptCost = 12
|
||||
|
||||
// sftpDummyBcryptHash burns bcrypt time on "username not found" so that
|
||||
// VerifySFTPLogin does not leak credential existence via timing, mirroring
|
||||
// userstore.dummyBcryptHash.
|
||||
const sftpDummyBcryptHash = "$2a$12$C6UzMDM.H6dfI/f/IKcEeO4TW/OZ/6PdTdSU0/eV1JCJXo.0DGvTa"
|
||||
|
||||
// ErrSFTPCredentialNotFound is returned when a credential lookup/revoke
|
||||
// targets a username or ID that doesn't exist (or doesn't belong to the
|
||||
// given tenant).
|
||||
var ErrSFTPCredentialNotFound = errors.New("storage: sftp credential not found")
|
||||
|
||||
// SFTPCredential is a per-tenant SFTP login. The password is never stored or
|
||||
// returned in plaintext after creation.
|
||||
type SFTPCredential struct {
|
||||
ID int64 `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
Username string `json:"username"`
|
||||
Active bool `json:"active"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
LastLoginAt *time.Time `json:"last_login_at,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Store) initSFTPCredentialsSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS sftp_credentials (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
password_hash TEXT NOT NULL,
|
||||
active BOOLEAN NOT NULL DEFAULT true,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
last_login_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_sftp_credentials_tenant ON sftp_credentials(tenant_id);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create sftp_credentials table: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateSFTPCredential creates a new SFTP credential for a tenant. The
|
||||
// caller-supplied plaintext password is bcrypt-hashed before storage and is
|
||||
// never persisted or retrievable again — the caller (the admin API handler)
|
||||
// must return it to the requester exactly once, in the create response.
|
||||
func (s *Store) CreateSFTPCredential(ctx context.Context, tenantID int64, username, password string) (*SFTPCredential, error) {
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte(password), sftpBcryptCost)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: sftp bcrypt: %w", err)
|
||||
}
|
||||
var c SFTPCredential
|
||||
err = s.db.QueryRow(ctx, `
|
||||
INSERT INTO sftp_credentials (tenant_id, username, password_hash, active, created_at)
|
||||
VALUES ($1, $2, $3, true, now())
|
||||
RETURNING id, tenant_id, username, active, created_at, last_login_at
|
||||
`, tenantID, username, string(hash),
|
||||
).Scan(&c.ID, &c.TenantID, &c.Username, &c.Active, &c.CreatedAt, &c.LastLoginAt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: create sftp credential: %w", err)
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// VerifySFTPLogin checks a username/password pair against sftp_credentials
|
||||
// and returns the credential (with tenant scope) on success. Inactive
|
||||
// credentials and wrong passwords both fail; unknown usernames burn a dummy
|
||||
// bcrypt compare to avoid a timing side-channel that would reveal which
|
||||
// usernames exist.
|
||||
func (s *Store) VerifySFTPLogin(ctx context.Context, username, password string) (*SFTPCredential, error) {
|
||||
var c SFTPCredential
|
||||
var hash string
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT id, tenant_id, username, password_hash, active, created_at, last_login_at
|
||||
FROM sftp_credentials WHERE username = $1
|
||||
`, username).Scan(&c.ID, &c.TenantID, &c.Username, &hash, &c.Active, &c.CreatedAt, &c.LastLoginAt)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
_ = bcrypt.CompareHashAndPassword([]byte(sftpDummyBcryptHash), []byte(password))
|
||||
return nil, fmt.Errorf("storage: sftp login: credential not found")
|
||||
}
|
||||
return nil, fmt.Errorf("storage: sftp login: %w", err)
|
||||
}
|
||||
if !c.Active {
|
||||
return nil, fmt.Errorf("storage: sftp login: credential revoked")
|
||||
}
|
||||
if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)); err != nil {
|
||||
return nil, fmt.Errorf("storage: sftp login: wrong password")
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// TouchSFTPLastLogin sets last_login_at = now() for a credential. Called
|
||||
// after a successful SFTP authentication.
|
||||
func (s *Store) TouchSFTPLastLogin(ctx context.Context, id int64) error {
|
||||
_, err := s.db.Exec(ctx, `UPDATE sftp_credentials SET last_login_at = now() WHERE id = $1`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// ListSFTPCredentials returns all SFTP credentials belonging to a tenant,
|
||||
// newest first. Password hashes are never included in the returned struct.
|
||||
func (s *Store) ListSFTPCredentials(ctx context.Context, tenantID int64) ([]SFTPCredential, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, tenant_id, username, active, created_at, last_login_at
|
||||
FROM sftp_credentials WHERE tenant_id = $1 ORDER BY created_at DESC
|
||||
`, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list sftp credentials: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]SFTPCredential, 0)
|
||||
for rows.Next() {
|
||||
var c SFTPCredential
|
||||
if err := rows.Scan(&c.ID, &c.TenantID, &c.Username, &c.Active, &c.CreatedAt, &c.LastLoginAt); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan sftp credential: %w", err)
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RevokeSFTPCredential deactivates (soft-deletes) a credential, scoped to the
|
||||
// owning tenant. Using active=false rather than a hard delete keeps the
|
||||
// row (and its audit trail via created_at/last_login_at) around for GoBD
|
||||
// traceability.
|
||||
func (s *Store) RevokeSFTPCredential(ctx context.Context, id, tenantID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE sftp_credentials SET active = false WHERE id = $1 AND tenant_id = $2`, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: revoke sftp credential: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrSFTPCredentialNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
// External document share-links (see migrations/009_shares.sql). A share is a
|
||||
// tenant-scoped, expiring, optionally password-protected public link to a
|
||||
// single document. The raw token is generated once (32 bytes crypto/rand,
|
||||
// base64url) and returned to the caller exactly once at creation time; only its
|
||||
// SHA-256 hash is ever persisted (token_hash). The public download endpoint
|
||||
// always looks a share up by token_hash, never by id.
|
||||
//
|
||||
// Shares are never hard-deleted: revoking only sets revoked_at/revoked_by, and
|
||||
// every access attempt (success or failure) is recorded in
|
||||
// document_share_accesses for GoBD traceability. The underlying file stays in
|
||||
// the WORM store and is streamed server-side; storage_path/content_hash are
|
||||
// never exposed to the public client.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
const shareBcryptCost = 12
|
||||
|
||||
// shareDummyBcryptHash burns bcrypt time when a share has no password but a
|
||||
// client nevertheless submits one, so password-protected and unprotected
|
||||
// shares are not trivially distinguishable by response timing.
|
||||
const shareDummyBcryptHash = "$2a$12$C6UzMDM.H6dfI/f/IKcEeO4TW/OZ/6PdTdSU0/eV1JCJXo.0DGvTa"
|
||||
|
||||
// Share lifecycle / validation errors.
|
||||
var (
|
||||
// ErrShareNotFound is returned when a share lookup (by id+tenant or by
|
||||
// token_hash) matches no row.
|
||||
ErrShareNotFound = errors.New("storage: share not found")
|
||||
// ErrShareRevoked is returned when a share has been revoked.
|
||||
ErrShareRevoked = errors.New("storage: share revoked")
|
||||
// ErrShareExpired is returned when a share is past its expires_at.
|
||||
ErrShareExpired = errors.New("storage: share expired")
|
||||
// ErrShareMaxReached is returned when a share hit its max_accesses cap.
|
||||
ErrShareMaxReached = errors.New("storage: share max accesses reached")
|
||||
// ErrShareBadPassword is returned when a required share password is
|
||||
// missing or wrong.
|
||||
ErrShareBadPassword = errors.New("storage: share password incorrect")
|
||||
)
|
||||
|
||||
// Share access-result constants (mirror the CHECK constraint on
|
||||
// document_share_accesses.result).
|
||||
const (
|
||||
ShareResultSuccess = "success"
|
||||
ShareResultExpired = "expired"
|
||||
ShareResultRevoked = "revoked"
|
||||
ShareResultMaxReached = "max_reached"
|
||||
ShareResultBadPassword = "bad_password"
|
||||
ShareResultRateLimited = "rate_limited"
|
||||
)
|
||||
|
||||
// DocumentShare is the public-safe view of a share row. token_hash and
|
||||
// password_hash are deliberately NOT part of this struct so they can never be
|
||||
// serialised into an API response.
|
||||
type DocumentShare struct {
|
||||
ID int64 `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
DocumentID int64 `json:"document_id"`
|
||||
CreatedBy int64 `json:"created_by"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
MaxAccesses *int `json:"max_accesses,omitempty"`
|
||||
AccessCount int `json:"access_count"`
|
||||
HasPassword bool `json:"has_password"`
|
||||
RevokedAt *time.Time `json:"revoked_at,omitempty"`
|
||||
RevokedBy *int64 `json:"revoked_by,omitempty"`
|
||||
// DocumentTitle is joined in for the listing endpoints; empty when not
|
||||
// selected.
|
||||
DocumentTitle string `json:"document_title,omitempty"`
|
||||
}
|
||||
|
||||
// CreateShareRequest holds the parameters for creating a share.
|
||||
type CreateShareRequest struct {
|
||||
TenantID int64
|
||||
DocumentID int64
|
||||
CreatedBy int64
|
||||
ExpiresAt time.Time
|
||||
MaxAccesses *int // nil = unlimited (until expiry)
|
||||
Password string // "" = no password protection
|
||||
}
|
||||
|
||||
// ResolvedShare carries the internal fields the public download flow needs but
|
||||
// which must never reach the client: password_hash plus the document's WORM
|
||||
// storage location. Its sensitive fields are unexported and reached only via
|
||||
// accessor methods / the Verify* helpers, so a handler cannot accidentally
|
||||
// serialise password_hash or storage_path into a response.
|
||||
type ResolvedShare struct {
|
||||
share DocumentShare
|
||||
passwordHash string
|
||||
storagePath string
|
||||
contentHash string
|
||||
DocumentTitle string
|
||||
}
|
||||
|
||||
// ShareID returns the share's id.
|
||||
func (rs *ResolvedShare) ShareID() int64 { return rs.share.ID }
|
||||
|
||||
// TenantID returns the owning tenant id.
|
||||
func (rs *ResolvedShare) TenantID() int64 { return rs.share.TenantID }
|
||||
|
||||
// DocumentID returns the shared document's id.
|
||||
func (rs *ResolvedShare) DocumentID() int64 { return rs.share.DocumentID }
|
||||
|
||||
// ExpiresAt returns the share's expiry.
|
||||
func (rs *ResolvedShare) ExpiresAt() time.Time { return rs.share.ExpiresAt }
|
||||
|
||||
// HasPassword reports whether the share is password-protected.
|
||||
func (rs *ResolvedShare) HasPassword() bool { return rs.passwordHash != "" }
|
||||
|
||||
// StoragePath returns the WORM path of the underlying file (server-side only).
|
||||
func (rs *ResolvedShare) StoragePath() string { return rs.storagePath }
|
||||
|
||||
func (s *Store) initSharesSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
-- No FK on tenant_id / created_by / revoked_by: consistent with the rest
|
||||
-- of the schema (plain BIGINT), because tenants/users are owned by other
|
||||
-- stores that initialise after storage.New() (see cmd/archivdms/main.go).
|
||||
-- document_id keeps its FK: documents is this store's own table.
|
||||
CREATE TABLE IF NOT EXISTS document_shares (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
token_hash TEXT NOT NULL UNIQUE,
|
||||
created_by BIGINT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
max_accesses INT,
|
||||
access_count INT NOT NULL DEFAULT 0,
|
||||
password_hash TEXT,
|
||||
revoked_at TIMESTAMPTZ,
|
||||
revoked_by BIGINT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_shares_document ON document_shares(document_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_shares_tenant ON document_shares(tenant_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_share_accesses (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
share_id BIGINT NOT NULL REFERENCES document_shares(id) ON DELETE CASCADE,
|
||||
accessed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
ip_address INET,
|
||||
user_agent TEXT,
|
||||
result TEXT NOT NULL CHECK (result IN ('success','expired','revoked','max_reached','bad_password','rate_limited'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_share_accesses_share ON document_share_accesses(share_id, accessed_at DESC);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create shares tables: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// hashShareToken returns the hex-encoded SHA-256 of a raw share token, the
|
||||
// value persisted in / looked up from document_shares.token_hash.
|
||||
func hashShareToken(token string) string {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// CreateShare inserts a new share for a document and returns the stored share
|
||||
// plus the raw (plaintext) token. The token is returned ONLY here and never
|
||||
// again — only its SHA-256 hash is persisted. The document must belong to the
|
||||
// tenant, otherwise ErrShareNotFound is returned.
|
||||
func (s *Store) CreateShare(ctx context.Context, req CreateShareRequest) (*DocumentShare, string, error) {
|
||||
// Ownership check: the document must belong to the tenant (IDOR guard).
|
||||
var owned bool
|
||||
if err := s.db.QueryRow(ctx, `
|
||||
SELECT EXISTS(SELECT 1 FROM documents WHERE id = $1 AND tenant_id = $2)
|
||||
`, req.DocumentID, req.TenantID).Scan(&owned); err != nil {
|
||||
return nil, "", fmt.Errorf("storage: check share document: %w", err)
|
||||
}
|
||||
if !owned {
|
||||
return nil, "", ErrShareNotFound
|
||||
}
|
||||
|
||||
rawBytes := make([]byte, 32)
|
||||
if _, err := rand.Read(rawBytes); err != nil {
|
||||
return nil, "", fmt.Errorf("storage: generate share token: %w", err)
|
||||
}
|
||||
token := base64.RawURLEncoding.EncodeToString(rawBytes)
|
||||
tokenHash := hashShareToken(token)
|
||||
|
||||
var passwordHash any
|
||||
if req.Password != "" {
|
||||
h, err := bcrypt.GenerateFromPassword([]byte(req.Password), shareBcryptCost)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("storage: share bcrypt: %w", err)
|
||||
}
|
||||
passwordHash = string(h)
|
||||
}
|
||||
|
||||
var d DocumentShare
|
||||
var pw *string
|
||||
err := s.db.QueryRow(ctx, `
|
||||
INSERT INTO document_shares (tenant_id, document_id, token_hash, created_by, expires_at, max_accesses, password_hash)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
RETURNING id, tenant_id, document_id, created_by, created_at, expires_at, max_accesses, access_count, password_hash, revoked_at, revoked_by
|
||||
`, req.TenantID, req.DocumentID, tokenHash, req.CreatedBy, req.ExpiresAt, req.MaxAccesses, passwordHash,
|
||||
).Scan(&d.ID, &d.TenantID, &d.DocumentID, &d.CreatedBy, &d.CreatedAt, &d.ExpiresAt, &d.MaxAccesses, &d.AccessCount, &pw, &d.RevokedAt, &d.RevokedBy)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("storage: create share: %w", err)
|
||||
}
|
||||
d.HasPassword = pw != nil
|
||||
return &d, token, nil
|
||||
}
|
||||
|
||||
// ListSharesForDocument returns all shares (including revoked ones — no hard
|
||||
// delete) for a document, scoped to tenant ownership, newest first.
|
||||
func (s *Store) ListSharesForDocument(ctx context.Context, documentID, tenantID int64) ([]DocumentShare, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, tenant_id, document_id, created_by, created_at, expires_at, max_accesses, access_count,
|
||||
(password_hash IS NOT NULL), revoked_at, revoked_by
|
||||
FROM document_shares
|
||||
WHERE document_id = $1 AND tenant_id = $2
|
||||
ORDER BY created_at DESC
|
||||
`, documentID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list document shares: %w", err)
|
||||
}
|
||||
return scanShares(rows)
|
||||
}
|
||||
|
||||
// ListSharesForTenant returns all shares of a tenant with the document title
|
||||
// joined in, newest first (domain_admin/superadmin overview).
|
||||
func (s *Store) ListSharesForTenant(ctx context.Context, tenantID int64) ([]DocumentShare, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT sh.id, sh.tenant_id, sh.document_id, sh.created_by, sh.created_at, sh.expires_at,
|
||||
sh.max_accesses, sh.access_count, (sh.password_hash IS NOT NULL), sh.revoked_at, sh.revoked_by,
|
||||
COALESCE(d.title, '')
|
||||
FROM document_shares sh
|
||||
LEFT JOIN documents d ON d.id = sh.document_id
|
||||
WHERE sh.tenant_id = $1
|
||||
ORDER BY sh.created_at DESC
|
||||
`, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list tenant shares: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]DocumentShare, 0)
|
||||
for rows.Next() {
|
||||
var d DocumentShare
|
||||
if err := rows.Scan(&d.ID, &d.TenantID, &d.DocumentID, &d.CreatedBy, &d.CreatedAt, &d.ExpiresAt,
|
||||
&d.MaxAccesses, &d.AccessCount, &d.HasPassword, &d.RevokedAt, &d.RevokedBy, &d.DocumentTitle); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan tenant share: %w", err)
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func scanShares(rows pgx.Rows) ([]DocumentShare, error) {
|
||||
defer rows.Close()
|
||||
out := make([]DocumentShare, 0)
|
||||
for rows.Next() {
|
||||
var d DocumentShare
|
||||
if err := rows.Scan(&d.ID, &d.TenantID, &d.DocumentID, &d.CreatedBy, &d.CreatedAt, &d.ExpiresAt,
|
||||
&d.MaxAccesses, &d.AccessCount, &d.HasPassword, &d.RevokedAt, &d.RevokedBy); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan share: %w", err)
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RevokeShare marks a share as revoked (never hard-deleted), scoped to tenant
|
||||
// ownership. Idempotent-ish: revoking an already-revoked share updates
|
||||
// revoked_at/revoked_by again but still succeeds. Returns ErrShareNotFound when
|
||||
// no share of that id belongs to the tenant.
|
||||
func (s *Store) RevokeShare(ctx context.Context, shareID, tenantID, revokedBy int64) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE document_shares SET revoked_at = now(), revoked_by = $3
|
||||
WHERE id = $1 AND tenant_id = $2
|
||||
`, shareID, tenantID, revokedBy)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: revoke share: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrShareNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResolveShareByToken looks a share up by the SHA-256 hash of the raw token
|
||||
// (never by id) and joins the document's storage location. Returns
|
||||
// ErrShareNotFound when no share matches.
|
||||
func (s *Store) ResolveShareByToken(ctx context.Context, token string) (*ResolvedShare, error) {
|
||||
tokenHash := hashShareToken(token)
|
||||
var rs ResolvedShare
|
||||
var pw *string
|
||||
err := s.db.QueryRow(ctx, `
|
||||
SELECT sh.id, sh.tenant_id, sh.document_id, sh.created_by, sh.created_at, sh.expires_at,
|
||||
sh.max_accesses, sh.access_count, sh.password_hash, sh.revoked_at, sh.revoked_by,
|
||||
COALESCE(d.title, ''), COALESCE(d.storage_path, ''), COALESCE(d.content_hash, '')
|
||||
FROM document_shares sh
|
||||
JOIN documents d ON d.id = sh.document_id
|
||||
WHERE sh.token_hash = $1
|
||||
`, tokenHash).Scan(&rs.share.ID, &rs.share.TenantID, &rs.share.DocumentID, &rs.share.CreatedBy,
|
||||
&rs.share.CreatedAt, &rs.share.ExpiresAt, &rs.share.MaxAccesses, &rs.share.AccessCount, &pw,
|
||||
&rs.share.RevokedAt, &rs.share.RevokedBy, &rs.DocumentTitle, &rs.storagePath, &rs.contentHash)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrShareNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("storage: resolve share by token: %w", err)
|
||||
}
|
||||
if pw != nil {
|
||||
rs.passwordHash = *pw
|
||||
rs.share.HasPassword = true
|
||||
}
|
||||
return &rs, nil
|
||||
}
|
||||
|
||||
// VerifyState applies the fixed check order (revoked -> expired ->
|
||||
// max_accesses) and returns the matching result string (for the access log)
|
||||
// plus the sentinel error. now is passed in so callers share one timestamp.
|
||||
func (rs *ResolvedShare) VerifyState(now time.Time) (string, error) {
|
||||
if rs.share.RevokedAt != nil {
|
||||
return ShareResultRevoked, ErrShareRevoked
|
||||
}
|
||||
if !now.Before(rs.share.ExpiresAt) {
|
||||
return ShareResultExpired, ErrShareExpired
|
||||
}
|
||||
if rs.share.MaxAccesses != nil && rs.share.AccessCount >= *rs.share.MaxAccesses {
|
||||
return ShareResultMaxReached, ErrShareMaxReached
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// VerifyPassword checks a submitted password against the share's stored bcrypt
|
||||
// hash. Runs a bcrypt comparison (dummy hash when the share has no password) to
|
||||
// avoid leaking, via timing, whether a share is protected.
|
||||
func (rs *ResolvedShare) VerifyPassword(password string) error {
|
||||
if rs.passwordHash == "" {
|
||||
// Burn comparable time so protected/unprotected shares look alike.
|
||||
_ = bcrypt.CompareHashAndPassword([]byte(shareDummyBcryptHash), []byte(password))
|
||||
return nil
|
||||
}
|
||||
if err := bcrypt.CompareHashAndPassword([]byte(rs.passwordHash), []byte(password)); err != nil {
|
||||
return ErrShareBadPassword
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IncrementShareAccess atomically bumps access_count, but only while the share
|
||||
// is still within its cap — the WHERE guard closes the race where two parallel
|
||||
// downloads could both pass the in-memory max check. Returns true when the
|
||||
// counter was incremented (i.e. the download may proceed).
|
||||
func (s *Store) IncrementShareAccess(ctx context.Context, shareID int64) (bool, error) {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE document_shares
|
||||
SET access_count = access_count + 1
|
||||
WHERE id = $1
|
||||
AND revoked_at IS NULL
|
||||
AND expires_at > now()
|
||||
AND (max_accesses IS NULL OR access_count < max_accesses)
|
||||
`, shareID)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("storage: increment share access: %w", err)
|
||||
}
|
||||
return tag.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
// LogShareAccess appends an access-attempt record. ip may be empty (stored as
|
||||
// NULL). Errors are returned so the caller can decide, but a logging failure
|
||||
// should never block the response path — the public handler logs-and-continues.
|
||||
func (s *Store) LogShareAccess(ctx context.Context, shareID int64, ip, userAgent, result string) error {
|
||||
var ipArg any
|
||||
if ip != "" {
|
||||
ipArg = ip
|
||||
}
|
||||
_, err := s.db.Exec(ctx, `
|
||||
INSERT INTO document_share_accesses (share_id, ip_address, user_agent, result)
|
||||
VALUES ($1, $2::inet, $3, $4)
|
||||
`, shareID, ipArg, userAgent, result)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: log share access: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
// Package storage is the PostgreSQL-backed metadata + file-blob store for
|
||||
// archivdms. It follows archivmail's internal/storage pattern (idempotent
|
||||
// initSchema() run at startup, no external migration tool — see
|
||||
// internal/storage/migrations/README.md for the documentation convention)
|
||||
// but the core model is `documents`, not `emails`.
|
||||
//
|
||||
// Multi-tenancy is applied at the application layer: every query filters
|
||||
// manually by tenant_id (no Postgres row-level security), consistent with
|
||||
// archivmail.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"archivdms/internal/index"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Config holds the configuration for initialising a Store.
|
||||
type Config struct {
|
||||
Dir string // base directory for document blob storage
|
||||
DSN string // PostgreSQL DSN
|
||||
RetentionDays int // default GoBD retention period in days (0 = no default lock)
|
||||
}
|
||||
|
||||
// Store is the document metadata store (PostgreSQL) plus a file-based blob
|
||||
// store for the underlying document files.
|
||||
type Store struct {
|
||||
dir string
|
||||
db *pgxpool.Pool
|
||||
retentionDays int
|
||||
|
||||
// indexer is the optional (nil-able) full-text search sync layer
|
||||
// (internal/index). When nil, all index sync calls are silent no-ops —
|
||||
// Postgres stays the single source of truth. Wired via SetIndexer.
|
||||
indexer index.TenantIndexer
|
||||
// logger is used only for best-effort index-sync warnings. May be nil.
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// SetIndexer wires the optional full-text search index (internal/index) into
|
||||
// the store, together with a logger for best-effort sync warnings. Both may be
|
||||
// nil (index disabled). Call once at startup, before serving requests.
|
||||
func (s *Store) SetIndexer(indexer index.TenantIndexer, logger *slog.Logger) {
|
||||
s.indexer = indexer
|
||||
s.logger = logger
|
||||
}
|
||||
|
||||
// New initialises the storage directory and connects to PostgreSQL, creating
|
||||
// the schema if needed.
|
||||
func New(cfg Config) (*Store, error) {
|
||||
for _, sub := range []string{"documents"} {
|
||||
if err := os.MkdirAll(filepath.Join(cfg.Dir, sub), 0o700); err != nil {
|
||||
return nil, fmt.Errorf("storage: mkdir %s: %w", sub, err)
|
||||
}
|
||||
}
|
||||
|
||||
s := &Store{dir: cfg.Dir, retentionDays: cfg.RetentionDays}
|
||||
|
||||
if cfg.DSN != "" {
|
||||
pool, err := pgxpool.New(context.Background(), cfg.DSN)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: db connect: %w", err)
|
||||
}
|
||||
s.db = pool
|
||||
if err := s.initSchema(context.Background()); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("storage: init schema: %w", err)
|
||||
}
|
||||
// Reminders (Wiedervorlage) schema — kept in its own file (reminders.go)
|
||||
// following the archivmail saved_searches.go pattern, but wired in here
|
||||
// so a single storage.New() call brings up the whole schema.
|
||||
if err := s.initReminderSchema(context.Background()); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("storage: init reminder schema: %w", err)
|
||||
}
|
||||
// SFTP credentials (internal/sftpserver) — kept in its own file
|
||||
// (sftp_credentials.go), wired in here like reminders above.
|
||||
if err := s.initSFTPCredentialsSchema(context.Background()); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("storage: init sftp credentials schema: %w", err)
|
||||
}
|
||||
// Group-resolved document ACL (permissions.go) — wired in here like the
|
||||
// schemas above so a single storage.New() brings up the whole schema.
|
||||
if err := s.initPermissionsSchema(context.Background()); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("storage: init permissions schema: %w", err)
|
||||
}
|
||||
// External document share-links (shares.go) — wired in here like the
|
||||
// schemas above so a single storage.New() brings up the whole schema.
|
||||
if err := s.initSharesSchema(context.Background()); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("storage: init shares schema: %w", err)
|
||||
}
|
||||
// Per-tenant API keys for the Buchhaltungs-Pull-API
|
||||
// (accounting_api_keys.go) — wired in here like the schemas above.
|
||||
if err := s.initAccountingAPIKeysSchema(context.Background()); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("storage: init accounting api keys schema: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Close releases the database connection pool (if any).
|
||||
func (s *Store) Close() {
|
||||
if s.db != nil {
|
||||
s.db.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// DocumentPath returns the on-disk path for a document's stored blob,
|
||||
// addressed by content hash (WORM-friendly: same content -> same path).
|
||||
func (s *Store) DocumentPath(contentHash string) string {
|
||||
return filepath.Join(s.dir, "documents", contentHash)
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
// ErrTaxonomyNotFound is returned when a tag/document_type/correspondent
|
||||
// lookup, update or delete does not match any row owned by the caller's
|
||||
// tenant.
|
||||
var ErrTaxonomyNotFound = errors.New("storage: taxonomy entity not found or not owned by tenant")
|
||||
|
||||
// ErrDuplicateTaxonomyName is returned when a tenant already has an entity
|
||||
// of the same kind with the same name (UNIQUE(tenant_id, name)).
|
||||
var ErrDuplicateTaxonomyName = errors.New("storage: entity with this name already exists for tenant")
|
||||
|
||||
// TaxonomyEntity is the shared shape of tags, document_types and
|
||||
// correspondents — structurally identical (see lazy-splashing-puppy plan),
|
||||
// kept as one Go struct/table-set with a `kind` selector rather than three
|
||||
// near-duplicate types.
|
||||
type TaxonomyEntity struct {
|
||||
ID int64 `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
Name string `json:"name"`
|
||||
Color string `json:"color,omitempty"`
|
||||
MatchAlgorithm string `json:"match_algorithm"`
|
||||
MatchPattern string `json:"match_pattern,omitempty"`
|
||||
CaseSensitive bool `json:"case_sensitive"`
|
||||
BarcodeValue string `json:"barcode_value,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// TaxonomyEntityRequest holds create/update parameters for a taxonomy entity.
|
||||
type TaxonomyEntityRequest struct {
|
||||
Name string
|
||||
Color string
|
||||
MatchAlgorithm string
|
||||
MatchPattern string
|
||||
CaseSensitive bool
|
||||
BarcodeValue string
|
||||
}
|
||||
|
||||
// taxonomyTable maps the three supported "kinds" to their table name. Kept
|
||||
// as an allowlist so a caller can never inject an arbitrary table name.
|
||||
func taxonomyTable(kind string) (string, error) {
|
||||
switch kind {
|
||||
case "tags":
|
||||
return "tags", nil
|
||||
case "document_types":
|
||||
return "document_types", nil
|
||||
case "correspondents":
|
||||
return "correspondents", nil
|
||||
default:
|
||||
return "", fmt.Errorf("storage: unknown taxonomy kind %q", kind)
|
||||
}
|
||||
}
|
||||
|
||||
// initTaxonomySchema creates the tags/document_types/correspondents/
|
||||
// document_tags tables plus the documents-table ALTERs (doc_type_id,
|
||||
// correspondent_id, barcode_values). Idempotent, called from
|
||||
// (*Store).initSchema. Documented (not executed) in
|
||||
// migrations/005_taxonomy.sql.
|
||||
func (s *Store) initTaxonomySchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
color TEXT,
|
||||
match_algorithm TEXT NOT NULL DEFAULT 'none' CHECK (match_algorithm IN ('none','any','all','exact','regex','fuzzy')),
|
||||
match_pattern TEXT,
|
||||
case_sensitive BOOLEAN NOT NULL DEFAULT false,
|
||||
barcode_value TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, name)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS document_types (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
color TEXT,
|
||||
match_algorithm TEXT NOT NULL DEFAULT 'none' CHECK (match_algorithm IN ('none','any','all','exact','regex','fuzzy')),
|
||||
match_pattern TEXT,
|
||||
case_sensitive BOOLEAN NOT NULL DEFAULT false,
|
||||
barcode_value TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, name)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS correspondents (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
color TEXT,
|
||||
match_algorithm TEXT NOT NULL DEFAULT 'none' CHECK (match_algorithm IN ('none','any','all','exact','regex','fuzzy')),
|
||||
match_pattern TEXT,
|
||||
case_sensitive BOOLEAN NOT NULL DEFAULT false,
|
||||
barcode_value TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, name)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS document_tags (
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
tag_id BIGINT NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (document_id, tag_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tags_tenant ON tags(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_types_tenant ON document_types(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_correspondents_tenant ON correspondents(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_tags_tag ON document_tags(tag_id);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_tags_tenant_barcode ON tags(tenant_id, barcode_value) WHERE barcode_value IS NOT NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_document_types_tenant_barcode ON document_types(tenant_id, barcode_value) WHERE barcode_value IS NOT NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_correspondents_tenant_barcode ON correspondents(tenant_id, barcode_value) WHERE barcode_value IS NOT NULL;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create taxonomy tables: %w", err)
|
||||
}
|
||||
|
||||
// documents-table ALTERs: doc_type_id/correspondent_id/barcode_values.
|
||||
// The old doc_type/correspondent free-text columns are NOT touched
|
||||
// (Bestandsschutz) — they stay in place, deprecated in favor of these.
|
||||
_, err = s.db.Exec(ctx, `
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS doc_type_id BIGINT REFERENCES document_types(id);
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS correspondent_id BIGINT REFERENCES correspondents(id);
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS barcode_values JSONB;
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: alter documents table for taxonomy: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func scanTaxonomyEntity(row interface {
|
||||
Scan(dest ...any) error
|
||||
}) (*TaxonomyEntity, error) {
|
||||
var e TaxonomyEntity
|
||||
if err := row.Scan(&e.ID, &e.TenantID, &e.Name, &e.Color, &e.MatchAlgorithm, &e.MatchPattern, &e.CaseSensitive, &e.BarcodeValue, &e.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &e, nil
|
||||
}
|
||||
|
||||
// CreateTaxonomyEntity inserts a new tag/document_type/correspondent row.
|
||||
func (s *Store) CreateTaxonomyEntity(ctx context.Context, kind string, tenantID int64, req TaxonomyEntityRequest) (*TaxonomyEntity, error) {
|
||||
table, err := taxonomyTable(kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
algo := req.MatchAlgorithm
|
||||
if algo == "" {
|
||||
algo = "none"
|
||||
}
|
||||
row := s.db.QueryRow(ctx, fmt.Sprintf(`
|
||||
INSERT INTO %s (tenant_id, name, color, match_algorithm, match_pattern, case_sensitive, barcode_value)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
RETURNING id, tenant_id, name, COALESCE(color, ''), match_algorithm, COALESCE(match_pattern, ''), case_sensitive, COALESCE(barcode_value, ''), created_at
|
||||
`, table), tenantID, req.Name, nullIfEmpty(req.Color), algo, nullIfEmpty(req.MatchPattern), req.CaseSensitive, nullIfEmpty(req.BarcodeValue))
|
||||
e, err := scanTaxonomyEntity(row)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return nil, ErrDuplicateTaxonomyName
|
||||
}
|
||||
return nil, fmt.Errorf("storage: create %s: %w", kind, err)
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// ListTaxonomyEntities returns all entities of the given kind for a tenant.
|
||||
func (s *Store) ListTaxonomyEntities(ctx context.Context, kind string, tenantID int64) ([]TaxonomyEntity, error) {
|
||||
table, err := taxonomyTable(kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.db.Query(ctx, fmt.Sprintf(`
|
||||
SELECT id, tenant_id, name, COALESCE(color, ''), match_algorithm, COALESCE(match_pattern, ''), case_sensitive, COALESCE(barcode_value, ''), created_at
|
||||
FROM %s WHERE tenant_id = $1 ORDER BY name ASC
|
||||
`, table), tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list %s: %w", kind, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]TaxonomyEntity, 0)
|
||||
for rows.Next() {
|
||||
e, err := scanTaxonomyEntity(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: scan %s: %w", kind, err)
|
||||
}
|
||||
out = append(out, *e)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListActiveMatchers returns all entities of the given kind for a tenant
|
||||
// whose match_algorithm is not 'none' — the candidate set the matching
|
||||
// engine runs against on ingest.
|
||||
func (s *Store) ListActiveMatchers(ctx context.Context, kind string, tenantID int64) ([]TaxonomyEntity, error) {
|
||||
table, err := taxonomyTable(kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.db.Query(ctx, fmt.Sprintf(`
|
||||
SELECT id, tenant_id, name, COALESCE(color, ''), match_algorithm, COALESCE(match_pattern, ''), case_sensitive, COALESCE(barcode_value, ''), created_at
|
||||
FROM %s WHERE tenant_id = $1 AND match_algorithm != 'none'
|
||||
`, table), tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list active matchers %s: %w", kind, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []TaxonomyEntity
|
||||
for rows.Next() {
|
||||
e, err := scanTaxonomyEntity(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: scan %s: %w", kind, err)
|
||||
}
|
||||
out = append(out, *e)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// GetTaxonomyEntityByBarcode looks up an entity by its barcode_value, scoped
|
||||
// to tenant. Returns ErrTaxonomyNotFound if none match.
|
||||
func (s *Store) GetTaxonomyEntityByBarcode(ctx context.Context, kind string, tenantID int64, barcodeValue string) (*TaxonomyEntity, error) {
|
||||
table, err := taxonomyTable(kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row := s.db.QueryRow(ctx, fmt.Sprintf(`
|
||||
SELECT id, tenant_id, name, COALESCE(color, ''), match_algorithm, COALESCE(match_pattern, ''), case_sensitive, COALESCE(barcode_value, ''), created_at
|
||||
FROM %s WHERE tenant_id = $1 AND barcode_value = $2
|
||||
`, table), tenantID, barcodeValue)
|
||||
e, err := scanTaxonomyEntity(row)
|
||||
if err != nil {
|
||||
return nil, ErrTaxonomyNotFound
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// UpdateTaxonomyEntity updates a tag/document_type/correspondent, scoped to
|
||||
// tenant ownership.
|
||||
func (s *Store) UpdateTaxonomyEntity(ctx context.Context, kind string, id, tenantID int64, req TaxonomyEntityRequest) (*TaxonomyEntity, error) {
|
||||
table, err := taxonomyTable(kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
algo := req.MatchAlgorithm
|
||||
if algo == "" {
|
||||
algo = "none"
|
||||
}
|
||||
row := s.db.QueryRow(ctx, fmt.Sprintf(`
|
||||
UPDATE %s SET name = $1, color = $2, match_algorithm = $3, match_pattern = $4, case_sensitive = $5, barcode_value = $6
|
||||
WHERE id = $7 AND tenant_id = $8
|
||||
RETURNING id, tenant_id, name, COALESCE(color, ''), match_algorithm, COALESCE(match_pattern, ''), case_sensitive, COALESCE(barcode_value, ''), created_at
|
||||
`, table), req.Name, nullIfEmpty(req.Color), algo, nullIfEmpty(req.MatchPattern), req.CaseSensitive, nullIfEmpty(req.BarcodeValue), id, tenantID)
|
||||
e, err := scanTaxonomyEntity(row)
|
||||
if err != nil {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return nil, ErrDuplicateTaxonomyName
|
||||
}
|
||||
return nil, ErrTaxonomyNotFound
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// DeleteTaxonomyEntity deletes a tag/document_type/correspondent, scoped to
|
||||
// tenant ownership.
|
||||
func (s *Store) DeleteTaxonomyEntity(ctx context.Context, kind string, id, tenantID int64) error {
|
||||
table, err := taxonomyTable(kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := s.db.Exec(ctx, fmt.Sprintf(`DELETE FROM %s WHERE id = $1 AND tenant_id = $2`, table), id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: delete %s: %w", kind, err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrTaxonomyNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AttachTag adds a document_tags row (idempotent — repeated attaches are a
|
||||
// no-op via ON CONFLICT). Ownership of both document and tag by tenantID
|
||||
// must be verified by the caller beforehand.
|
||||
func (s *Store) AttachTag(ctx context.Context, documentID, tagID int64) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
INSERT INTO document_tags (document_id, tag_id) VALUES ($1, $2)
|
||||
ON CONFLICT (document_id, tag_id) DO NOTHING
|
||||
`, documentID, tagID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: attach tag: %w", err)
|
||||
}
|
||||
// The document's tag set changed -> its tag_grants layer may differ.
|
||||
if err := s.RecomputeVisibility(ctx, documentID); err != nil {
|
||||
return fmt.Errorf("storage: recompute visibility after attach tag: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DetachTag removes a document_tags row.
|
||||
func (s *Store) DetachTag(ctx context.Context, documentID, tagID int64) error {
|
||||
_, err := s.db.Exec(ctx, `DELETE FROM document_tags WHERE document_id = $1 AND tag_id = $2`, documentID, tagID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: detach tag: %w", err)
|
||||
}
|
||||
if err := s.RecomputeVisibility(ctx, documentID); err != nil {
|
||||
return fmt.Errorf("storage: recompute visibility after detach tag: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListDocumentTags returns the tags attached to a document.
|
||||
func (s *Store) ListDocumentTags(ctx context.Context, documentID, tenantID int64) ([]TaxonomyEntity, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT t.id, t.tenant_id, t.name, COALESCE(t.color, ''), t.match_algorithm, COALESCE(t.match_pattern, ''), t.case_sensitive, COALESCE(t.barcode_value, ''), t.created_at
|
||||
FROM tags t
|
||||
JOIN document_tags dt ON dt.tag_id = t.id
|
||||
WHERE dt.document_id = $1 AND t.tenant_id = $2
|
||||
ORDER BY t.name ASC
|
||||
`, documentID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list document tags: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]TaxonomyEntity, 0)
|
||||
for rows.Next() {
|
||||
e, err := scanTaxonomyEntity(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: scan document tag: %w", err)
|
||||
}
|
||||
out = append(out, *e)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SetDocumentBarcodeValues stores the raw barcode payloads detected during
|
||||
// ingest on documents.barcode_values (JSONB array), regardless of whether
|
||||
// they matched any taxonomy entity — kept for GoBD-Nachvollziehbarkeit.
|
||||
func (s *Store) SetDocumentBarcodeValues(ctx context.Context, documentID, tenantID int64, values []string) error {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
b, err := json.Marshal(values)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: marshal barcode values: %w", err)
|
||||
}
|
||||
_, err = s.db.Exec(ctx, `UPDATE documents SET barcode_values = $1 WHERE id = $2 AND tenant_id = $3`, b, documentID, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: set document barcode values: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetDocumentDocType sets documents.doc_type_id, scoped to tenant.
|
||||
func (s *Store) SetDocumentDocType(ctx context.Context, documentID, tenantID, docTypeID int64) error {
|
||||
_, err := s.db.Exec(ctx, `UPDATE documents SET doc_type_id = $1 WHERE id = $2 AND tenant_id = $3`, docTypeID, documentID, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: set document doc_type_id: %w", err)
|
||||
}
|
||||
// The document's type changed -> its document_type_grants layer may differ.
|
||||
if err := s.RecomputeVisibility(ctx, documentID); err != nil {
|
||||
return fmt.Errorf("storage: recompute visibility after set doc_type: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetDocumentCorrespondent sets documents.correspondent_id, scoped to tenant.
|
||||
func (s *Store) SetDocumentCorrespondent(ctx context.Context, documentID, tenantID, correspondentID int64) error {
|
||||
_, err := s.db.Exec(ctx, `UPDATE documents SET correspondent_id = $1 WHERE id = $2 AND tenant_id = $3`, correspondentID, documentID, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: set document correspondent_id: %w", err)
|
||||
}
|
||||
// correspondent is not part of the ACL, so no RecomputeVisibility runs
|
||||
// here — sync the index directly. Best-effort.
|
||||
s.SyncIndex(ctx, documentID)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// Trash + staged deletion (Papierkorb + gestaffeltes Löschkonzept).
|
||||
//
|
||||
// A document with deleted_at IS NOT NULL sits in the trash. Its WORM file on
|
||||
// disk stays physically untouched (chmod 0440) until a delete request reaches
|
||||
// status='executed'. Final deletion requires a two-person rule (Vier-Augen-
|
||||
// Prinzip): one user requests it, a *different* user with domain_admin role
|
||||
// confirms it. Both the request and the confirm re-check retain_until, since
|
||||
// time may have passed or a new retention lock may have appeared in between.
|
||||
//
|
||||
// On execution the physical file is removed but the DB row is kept as a
|
||||
// tombstone (GoBD Nachvollziehbarkeit): storage_path and ocr_text are cleared,
|
||||
// content_hash is retained so the former existence + integrity fingerprint of
|
||||
// the document stays provable.
|
||||
|
||||
var (
|
||||
// ErrDocumentNotInTrash is returned when a trash operation targets a
|
||||
// document that is not (or no longer) soft-deleted for the tenant.
|
||||
ErrDocumentNotInTrash = errors.New("storage: document not found in trash for tenant")
|
||||
|
||||
// ErrAlreadyInTrash is returned by SoftDeleteDocument when the document is
|
||||
// already soft-deleted.
|
||||
ErrAlreadyInTrash = errors.New("storage: document already in trash")
|
||||
|
||||
// ErrDeleteRequestExists is returned when a pending delete request already
|
||||
// exists for the document.
|
||||
ErrDeleteRequestExists = errors.New("storage: a pending delete request already exists for this document")
|
||||
|
||||
// ErrDeleteRequestNotFound is returned when a delete request lookup does
|
||||
// not match a pending request owned by the tenant/document.
|
||||
ErrDeleteRequestNotFound = errors.New("storage: delete request not found or not pending")
|
||||
|
||||
// ErrSelfConfirm is returned when the confirming user is the same as the
|
||||
// requesting user (Vier-Augen-Prinzip violation).
|
||||
ErrSelfConfirm = errors.New("storage: delete request must be confirmed by a different user")
|
||||
|
||||
// ErrRetentionActive is returned when retain_until still blocks final
|
||||
// deletion (retain_until IS NOT NULL AND retain_until >= now()).
|
||||
ErrRetentionActive = errors.New("storage: document is under retention and cannot be finally deleted")
|
||||
)
|
||||
|
||||
// DeleteRequest is a single staged-deletion request row.
|
||||
type DeleteRequest struct {
|
||||
ID int64 `json:"id"`
|
||||
DocumentID int64 `json:"document_id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
RequestedBy int64 `json:"requested_by"`
|
||||
RequestedAt time.Time `json:"requested_at"`
|
||||
ConfirmedBy *int64 `json:"confirmed_by,omitempty"`
|
||||
ConfirmedAt *time.Time `json:"confirmed_at,omitempty"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
// TrashDocument is a soft-deleted document plus its trash metadata.
|
||||
type TrashDocument struct {
|
||||
Document
|
||||
DeletedAt *time.Time `json:"deleted_at,omitempty"`
|
||||
DeletedBy *int64 `json:"deleted_by,omitempty"`
|
||||
}
|
||||
|
||||
// ExecutedDelete carries the result of a confirmed + executed final deletion,
|
||||
// used by the handler to build the two audit entries (requester + confirmer)
|
||||
// and, crucially, the GoBD-taugliche Löschprotokoll (ecoDMS-Muster): who
|
||||
// requested, who confirmed, when, which document (title + content_hash as the
|
||||
// tamper-evident fingerprint of the now-removed WORM file) and the retention
|
||||
// state at execution time (RetainUntil, expected nil or already elapsed).
|
||||
type ExecutedDelete struct {
|
||||
RequestID int64
|
||||
RequestedBy int64
|
||||
RequestedAt time.Time
|
||||
StoragePath string
|
||||
ContentHash string
|
||||
Title string
|
||||
RetainUntil *time.Time
|
||||
}
|
||||
|
||||
func (s *Store) initTrashSchema(ctx context.Context) error {
|
||||
_, err := s.db.Exec(ctx, `
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||
ALTER TABLE documents ADD COLUMN IF NOT EXISTS deleted_by BIGINT;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_delete_requests (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
document_id BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
tenant_id BIGINT NOT NULL,
|
||||
requested_by BIGINT NOT NULL,
|
||||
requested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
confirmed_by BIGINT,
|
||||
confirmed_at TIMESTAMPTZ,
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending','confirmed','executed','cancelled','blocked_retention')),
|
||||
UNIQUE(document_id, status)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ddr_tenant_status ON document_delete_requests(tenant_id, status);
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: create trash schema: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SoftDeleteDocument moves a document into the trash (sets deleted_at/deleted_by).
|
||||
// The WORM file is left untouched. Returns ErrAlreadyInTrash if the document is
|
||||
// already trashed, ErrDocumentNotInTrash if it does not exist for the tenant.
|
||||
func (s *Store) SoftDeleteDocument(ctx context.Context, id, tenantID, userID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE documents SET deleted_at = now(), deleted_by = $3, updated_at = now()
|
||||
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
|
||||
`, id, tenantID, userID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: soft delete document: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
// Distinguish "does not exist" from "already trashed" for a clearer error.
|
||||
var deleted bool
|
||||
qerr := s.db.QueryRow(ctx,
|
||||
`SELECT deleted_at IS NOT NULL FROM documents WHERE id = $1 AND tenant_id = $2`,
|
||||
id, tenantID).Scan(&deleted)
|
||||
if qerr != nil {
|
||||
return ErrDocumentNotInTrash
|
||||
}
|
||||
if deleted {
|
||||
return ErrAlreadyInTrash
|
||||
}
|
||||
return ErrDocumentNotInTrash
|
||||
}
|
||||
// Trashed documents must not surface in search. The WORM file and DB row
|
||||
// stay put (restore re-indexes); only the index entry is dropped.
|
||||
// Best-effort.
|
||||
s.DeleteFromIndex(ctx, id, tenantID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListTrash returns all soft-deleted documents for a tenant, newest deletion
|
||||
// first.
|
||||
func (s *Store) ListTrash(ctx context.Context, tenantID int64) ([]TrashDocument, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, tenant_id, title, COALESCE(doc_type, ''), COALESCE(correspondent, ''), doc_type_id, correspondent_id, storage_path, content_hash,
|
||||
COALESCE(ocr_text, ''), retain_until, COALESCE(source, ''), COALESCE(source_ref, ''), created_at, updated_at, deleted_at, deleted_by
|
||||
FROM documents WHERE tenant_id = $1 AND deleted_at IS NOT NULL ORDER BY deleted_at DESC
|
||||
`, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list trash: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]TrashDocument, 0)
|
||||
for rows.Next() {
|
||||
var d TrashDocument
|
||||
if err := rows.Scan(&d.ID, &d.TenantID, &d.Title, &d.DocType, &d.Correspondent, &d.DocTypeID, &d.CorrespondentID, &d.StoragePath, &d.ContentHash,
|
||||
&d.OCRText, &d.RetainUntil, &d.Source, &d.SourceRef, &d.CreatedAt, &d.UpdatedAt, &d.DeletedAt, &d.DeletedBy); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan trash document: %w", err)
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RestoreDocument brings a document back out of the trash: clears deleted_at/
|
||||
// deleted_by and cancels any pending delete request. Returns
|
||||
// ErrDocumentNotInTrash if the document is not currently trashed.
|
||||
func (s *Store) RestoreDocument(ctx context.Context, id, tenantID, userID int64) error {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: restore begin: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE documents SET deleted_at = NULL, deleted_by = NULL, updated_at = now()
|
||||
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NOT NULL
|
||||
`, id, tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: restore document: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrDocumentNotInTrash
|
||||
}
|
||||
|
||||
// Cancel any pending request so a later trash cycle can start clean.
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE document_delete_requests SET status = 'cancelled', confirmed_by = $3, confirmed_at = now()
|
||||
WHERE document_id = $1 AND tenant_id = $2 AND status = 'pending'
|
||||
`, id, tenantID, userID); err != nil {
|
||||
return fmt.Errorf("storage: cancel pending requests on restore: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("storage: restore commit: %w", err)
|
||||
}
|
||||
// Back out of the trash -> re-index so it is findable again. Best-effort.
|
||||
s.SyncIndex(ctx, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateDeleteRequest records a final-deletion request (User A). It re-checks
|
||||
// retention: if retain_until still blocks deletion, a row with
|
||||
// status='blocked_retention' is written (for Nachvollziehbarkeit of the attempt)
|
||||
// and ErrRetentionActive is returned. Otherwise a 'pending' request is created.
|
||||
// The document must currently be in the trash.
|
||||
func (s *Store) CreateDeleteRequest(ctx context.Context, docID, tenantID, userID int64) (*DeleteRequest, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: delete request begin: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var retainUntil *time.Time
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT retain_until FROM documents
|
||||
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NOT NULL
|
||||
FOR UPDATE
|
||||
`, docID, tenantID).Scan(&retainUntil)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrDocumentNotInTrash
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: load document for delete request: %w", err)
|
||||
}
|
||||
|
||||
if retentionActive(retainUntil) {
|
||||
req, berr := upsertBlockedRequest(ctx, tx, docID, tenantID, userID)
|
||||
if berr != nil {
|
||||
return nil, berr
|
||||
}
|
||||
if cerr := tx.Commit(ctx); cerr != nil {
|
||||
return nil, fmt.Errorf("storage: delete request commit (blocked): %w", cerr)
|
||||
}
|
||||
return req, ErrRetentionActive
|
||||
}
|
||||
|
||||
var req DeleteRequest
|
||||
err = tx.QueryRow(ctx, `
|
||||
INSERT INTO document_delete_requests (document_id, tenant_id, requested_by, status)
|
||||
VALUES ($1, $2, $3, 'pending')
|
||||
ON CONFLICT (document_id, status) DO NOTHING
|
||||
RETURNING id, document_id, tenant_id, requested_by, requested_at, confirmed_by, confirmed_at, status
|
||||
`, docID, tenantID, userID).Scan(&req.ID, &req.DocumentID, &req.TenantID, &req.RequestedBy, &req.RequestedAt, &req.ConfirmedBy, &req.ConfirmedAt, &req.Status)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrDeleteRequestExists
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: insert delete request: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, fmt.Errorf("storage: delete request commit: %w", err)
|
||||
}
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
// upsertBlockedRequest records/refreshes a blocked_retention request row.
|
||||
func upsertBlockedRequest(ctx context.Context, tx pgx.Tx, docID, tenantID, userID int64) (*DeleteRequest, error) {
|
||||
var req DeleteRequest
|
||||
err := tx.QueryRow(ctx, `
|
||||
INSERT INTO document_delete_requests (document_id, tenant_id, requested_by, status)
|
||||
VALUES ($1, $2, $3, 'blocked_retention')
|
||||
ON CONFLICT (document_id, status)
|
||||
DO UPDATE SET requested_by = EXCLUDED.requested_by, requested_at = now()
|
||||
RETURNING id, document_id, tenant_id, requested_by, requested_at, confirmed_by, confirmed_at, status
|
||||
`, docID, tenantID, userID).Scan(&req.ID, &req.DocumentID, &req.TenantID, &req.RequestedBy, &req.RequestedAt, &req.ConfirmedBy, &req.ConfirmedAt, &req.Status)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: record blocked_retention request: %w", err)
|
||||
}
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
// ListDeleteRequests returns all delete requests for a document (status/history),
|
||||
// scoped to tenant, newest first.
|
||||
func (s *Store) ListDeleteRequests(ctx context.Context, docID, tenantID int64) ([]DeleteRequest, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, document_id, tenant_id, requested_by, requested_at, confirmed_by, confirmed_at, status
|
||||
FROM document_delete_requests
|
||||
WHERE document_id = $1 AND tenant_id = $2
|
||||
ORDER BY requested_at DESC, id DESC
|
||||
`, docID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: list delete requests: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]DeleteRequest, 0)
|
||||
for rows.Next() {
|
||||
var req DeleteRequest
|
||||
if err := rows.Scan(&req.ID, &req.DocumentID, &req.TenantID, &req.RequestedBy, &req.RequestedAt, &req.ConfirmedBy, &req.ConfirmedAt, &req.Status); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan delete request: %w", err)
|
||||
}
|
||||
out = append(out, req)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// CancelDeleteRequest withdraws a pending delete request (status='cancelled').
|
||||
func (s *Store) CancelDeleteRequest(ctx context.Context, docID, reqID, tenantID, userID int64) error {
|
||||
tag, err := s.db.Exec(ctx, `
|
||||
UPDATE document_delete_requests
|
||||
SET status = 'cancelled', confirmed_by = $4, confirmed_at = now()
|
||||
WHERE id = $1 AND document_id = $2 AND tenant_id = $3 AND status = 'pending'
|
||||
`, reqID, docID, tenantID, userID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: cancel delete request: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrDeleteRequestNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ConfirmDeleteRequest confirms a pending request (User B) and, if retention
|
||||
// still allows it, executes the final deletion: it removes the physical WORM
|
||||
// file and turns the DB row into a tombstone (storage_path/ocr_text cleared,
|
||||
// content_hash kept). The confirmer must differ from the requester
|
||||
// (ErrSelfConfirm) and must be enforced to hold domain_admin by the caller.
|
||||
//
|
||||
// If retain_until now blocks deletion, the request is moved to
|
||||
// status='blocked_retention' (recording the attempt) and ErrRetentionActive is
|
||||
// returned. On any other outcome the returned *ExecutedDelete is nil.
|
||||
func (s *Store) ConfirmDeleteRequest(ctx context.Context, docID, reqID, tenantID, confirmerID int64) (*ExecutedDelete, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: confirm begin: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var (
|
||||
requestedBy int64
|
||||
requestedAt time.Time
|
||||
)
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT requested_by, requested_at FROM document_delete_requests
|
||||
WHERE id = $1 AND document_id = $2 AND tenant_id = $3 AND status = 'pending'
|
||||
FOR UPDATE
|
||||
`, reqID, docID, tenantID).Scan(&requestedBy, &requestedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrDeleteRequestNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: load delete request: %w", err)
|
||||
}
|
||||
|
||||
if confirmerID == requestedBy {
|
||||
// Vier-Augen-Prinzip: refuse and leave the request pending.
|
||||
return nil, ErrSelfConfirm
|
||||
}
|
||||
|
||||
var (
|
||||
retainUntil *time.Time
|
||||
storagePath string
|
||||
contentHash string
|
||||
title string
|
||||
)
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT retain_until, storage_path, COALESCE(content_hash, ''), COALESCE(title, '') FROM documents
|
||||
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NOT NULL
|
||||
FOR UPDATE
|
||||
`, docID, tenantID).Scan(&retainUntil, &storagePath, &contentHash, &title)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrDocumentNotInTrash
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: load document for confirm: %w", err)
|
||||
}
|
||||
|
||||
if retentionActive(retainUntil) {
|
||||
// Record the blocked attempt on the (previously pending) request.
|
||||
if _, uerr := tx.Exec(ctx, `
|
||||
UPDATE document_delete_requests
|
||||
SET status = 'blocked_retention', confirmed_by = $2, confirmed_at = now()
|
||||
WHERE id = $1
|
||||
`, reqID, confirmerID); uerr != nil {
|
||||
return nil, fmt.Errorf("storage: record blocked confirm: %w", uerr)
|
||||
}
|
||||
if cerr := tx.Commit(ctx); cerr != nil {
|
||||
return nil, fmt.Errorf("storage: confirm commit (blocked): %w", cerr)
|
||||
}
|
||||
return nil, ErrRetentionActive
|
||||
}
|
||||
|
||||
// Mark executed and tombstone the document (keep content_hash).
|
||||
if _, uerr := tx.Exec(ctx, `
|
||||
UPDATE document_delete_requests
|
||||
SET status = 'executed', confirmed_by = $2, confirmed_at = now()
|
||||
WHERE id = $1
|
||||
`, reqID, confirmerID); uerr != nil {
|
||||
return nil, fmt.Errorf("storage: mark request executed: %w", uerr)
|
||||
}
|
||||
if _, uerr := tx.Exec(ctx, `
|
||||
UPDATE documents
|
||||
SET storage_path = '', ocr_text = NULL, updated_at = now()
|
||||
WHERE id = $1 AND tenant_id = $2
|
||||
`, docID, tenantID); uerr != nil {
|
||||
return nil, fmt.Errorf("storage: tombstone document: %w", uerr)
|
||||
}
|
||||
|
||||
// Remove the physical WORM file. The store directory is writable by the
|
||||
// process even though the file itself is 0440, so os.Remove succeeds. A
|
||||
// missing file is tolerated (already gone) — anything else aborts the tx so
|
||||
// the DB stays consistent with the filesystem.
|
||||
if storagePath != "" {
|
||||
if rerr := os.Remove(storagePath); rerr != nil && !os.IsNotExist(rerr) {
|
||||
return nil, fmt.Errorf("storage: remove WORM file: %w", rerr)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, fmt.Errorf("storage: confirm commit: %w", err)
|
||||
}
|
||||
// GoBD-critical: a finally-deleted document must not remain findable in the
|
||||
// search index, even though the search endpoint only lands in a later
|
||||
// phase. Best-effort, but consistency is enforced from day one.
|
||||
s.DeleteFromIndex(ctx, docID, tenantID)
|
||||
return &ExecutedDelete{
|
||||
RequestID: reqID,
|
||||
RequestedBy: requestedBy,
|
||||
RequestedAt: requestedAt,
|
||||
StoragePath: storagePath,
|
||||
ContentHash: contentHash,
|
||||
Title: title,
|
||||
RetainUntil: retainUntil,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// retentionActive reports whether retain_until still blocks final deletion:
|
||||
// a NULL retain_until never blocks; a date in the future (>= today) blocks.
|
||||
func retentionActive(retainUntil *time.Time) bool {
|
||||
if retainUntil == nil {
|
||||
return false
|
||||
}
|
||||
// retain_until is a DATE; deletion is allowed once it is strictly in the
|
||||
// past (retain_until < now()). Equal-to-today still blocks.
|
||||
return !retainUntil.Before(time.Now())
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user