scripte/archivdms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FDN-01: repository & projektgerüst

Browse Source 
Git-Repository für bestehenden archivdms-Code initialisiert, Branch-/Commit-Konvention (feature/<ticket>-<slug>-Branches, Ticket-Prefix in Commit-Nachricht) etabliert.
This commit is contained in:
patrick
2026-08-11 21:27:53 +02:00
parent 40ed80da71
commit 9a24ea29e1
274 changed files with 53708 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/audit/audit.go
+507
View File
@@ -0,0 +1,507 @@
// Package audit is a PostgreSQL-backed, append-only audit log, ported 1:1
// from archivmail's internal/audit pattern (including the DB-level
// immutability trigger and the tamper-evident JSON-Lines mirror file), with
// the mail-specific fields (mail_id, query) replaced by a generic document_id
// so the log fits archivdms's document-centric core model.
package audit
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
// Event type constants.
const (
EventLogin = "login"
EventLogout = "logout"
EventUserMgmt = "user_mgmt"
EventTenantMgmt = "tenant_mgmt"
// EventDocumentCreate/Update/Delete cover document lifecycle changes.
EventDocumentCreate = "document_create"
EventDocumentUpdate = "document_update"
EventDocumentDelete = "document_delete"
// EventDocumentReprocessed records a re-processing run on an already-archived
// document (POST /api/documents/{id}/reprocess): OCR is re-run on the stored
// WORM file and ocr_text refreshed, followed by best-effort auto-assignment
// and on_upload workflows. The file itself stays untouched (WORM).
EventDocumentReprocessed = "document_reprocessed"
// EventDocumentProcessed records the asynchronous post-upload processing
// run of the tenant job queue (internal/jobqueue -> api.ProcessDocumentJob):
// OCR extraction, title/belegdatum derivation, taxonomy auto-assignment and
// on_upload workflows on an already-staged, already-archived document.
// Logged on success AND failure (retry/backoff attempts included) so the
// GoBD trail covers the whole ingest chain, not just the synchronous
// staging step (EventDocumentCreate). The WORM file is never touched.
EventDocumentProcessed = "document_processed"
// Manual taxonomy assignment on a single document (doc_type/correspondent).
// Auto-assignment during upload is covered by EventDocumentCreate.
EventDocTypeSet = "document_doctype_set"
EventCorrespondentSet = "document_correspondent_set"
// Trash / staged deletion workflow (Papierkorb + Vier-Augen-Prinzip).
// Each phase produces its own append-only entry; confirm/execute emit
// separate entries for requester (User A) and confirmer (User B).
EventDocumentTrash = "document_trash" // soft-delete into trash
EventDocumentRestore = "document_restore" // restored from trash
EventDocumentDeleteRequest = "document_delete_request" // final-deletion requested (User A)
EventDocumentDeleteConfirm = "document_delete_confirm" // final-deletion confirmed (User B)
EventDocumentDeleteExecute = "document_delete_execute" // WORM file removed, tombstone kept
EventDocumentDeleteBlocked = "document_delete_blocked_retention" // blocked by retain_until
// Reminder ("Wiedervorlage") events.
EventReminderCreate = "reminder_create"
EventReminderStatusChange = "reminder_status_change"
EventReminderDelete = "reminder_delete"
EventReminderNotify = "reminder_notify" // cron: due-date notification sent
// SFTP credential lifecycle + login events (internal/sftpserver,
// internal/api/sftp_handlers.go).
EventSFTPCredentialCreate = "sftp_credential_create"
EventSFTPCredentialRevoke = "sftp_credential_revoke"
EventSFTPLogin = "sftp_login" // logged on every attempt, success and failure
// Permission model (group-resolved document ACL, internal/storage/permissions.go,
// internal/api/permission_handlers.go). Covers group/member changes and all
// three grant layers (document-type / tag / per-document, incl. 'deny').
EventPermissionGrantChanged = "permission_grant_changed"
// External document share-links (internal/storage/shares.go,
// internal/api/share_handlers.go + public_share_handlers.go). Create/Revoke
// are authenticated tenant actions; Accessed is logged on the public
// download endpoint (success and failure — see also the per-attempt
// document_share_accesses table for the full public-access trail).
EventShareCreated = "share_created"
EventShareRevoked = "share_revoked"
EventShareAccessed = "share_accessed"
// LDAP directory integration (internal/ldapstore, internal/ldapauth,
// internal/api/ldap_handlers.go). ConfigChanged covers create/update/delete
// and the test action; LoginSuccess/Failed are logged on every LDAP bind
// attempt (incl. JIT provisioning); RoleSync records a role change applied by
// group membership re-synchronisation (including downgrades).
EventLdapConfigChanged = "ldap_config_changed"
EventLdapLoginSuccess = "ldap_login_success"
EventLdapLoginFailed = "ldap_login_failed"
EventLdapRoleSync = "ldap_role_sync"
// EventOllamaConfigUpdate records a change to a tenant's external-Ollama
// connection config (GET/PUT /api/ollama-config). Logged on every attempt,
// success and failure (GoBD-Nachvollziehbarkeit).
EventOllamaConfigUpdate = "ollama_config_update"
// Classification templates (Klassifizierungsvorlagen,
// internal/storage/classification_templates.go,
// internal/api/classification_template_handlers.go). Create/Update/Delete
// cover template administration (incl. tag / field-default bulk replace);
// Applied records applying a template to a document — logged on success and
// failure, and also when a retain_until shortening attempt was rejected
// (RetainUntilBlocked) so GoBD traceability shows the rejection explicitly.
EventTemplateCreate = "classification_template_create"
EventTemplateUpdate = "classification_template_update"
EventTemplateDelete = "classification_template_delete"
EventTemplateApplied = "classification_template_applied"
// Workflows / Consumption-Regeln (internal/storage/workflows.go,
// internal/api/workflow_handlers.go). Create/Update/Delete cover workflow
// administration (incl. action bulk replace and dry-run test). EventWorkflowRun
// records a manual (test-endpoint) or automatic (on_upload) evaluation. Note
// automatic runs are additionally recorded document-scoped in the
// workflow_runs table for GoBD reproducibility.
EventWorkflowCreate = "workflow_create"
EventWorkflowUpdate = "workflow_update"
EventWorkflowDelete = "workflow_delete"
EventWorkflowRun = "workflow_run"
// Heuristische Metadaten-Vorschläge (internal/storage/metadata_suggestions.go,
// internal/api/metadata_suggestion_handlers.go). Records a (non-binding)
// suggestion run for a document; accepting a suggested field goes through the
// normal edit endpoints, not through this event.
EventSuggestionGenerated = "metadata_suggestion_generated"
// Freitext-Notizen pro Dokument (internal/storage/document_notes.go,
// internal/api/document_note_handlers.go). Create/Delete cover the note
// lifecycle; pure reads (listing notes) are not audited, consistent with the
// rest of this project. Notes are hard-deleted (not GoBD documents), but the
// deletion itself is still recorded for Nachvollziehbarkeit.
EventNoteCreate = "document_note_create"
EventNoteDelete = "document_note_delete"
// EventMLRetrain records a Naive-Bayes classifier retraining run
// (internal/classifier, cmd/archivdms/cmd_classify_retrain.go, cron-driven).
// Logged once per tenant per retrain, success and failure — a per-tenant
// failure is isolated and does not block the other tenants. Detail carries
// the per-kind document counts / skip reasons for GoBD-Nachvollziehbarkeit.
EventMLRetrain = "ml_classifier_retrain"
// Gespeicherte Suchansichten (SavedViews, Paperless-ngx inspiriert —
// internal/storage/saved_views.go, internal/api/saved_view_handlers.go).
// Create/Update/Delete cover a user's named, reusable search/filter view.
// Views can be private (own) or shared tenant-wide (is_shared); only the
// creator may update or delete a view. Pure reads (listing views) are not
// audited, consistent with the rest of this project.
EventSavedViewCreate = "saved_view_create"
EventSavedViewUpdate = "saved_view_update"
EventSavedViewDelete = "saved_view_delete"
// EventRetentionApplied records a batch run of the GoBD retention-rules
// engine (internal/storage/retention_rules.go ApplyRetentionRules,
// cmd/archivdms/cmd_retention_apply.go, cron-driven). One summary entry per
// run per tenant (tenant + count of documents whose retain_until was
// computed and set), NOT per document — batch-summary style to avoid audit
// log spam. Logged on success and failure.
EventRetentionApplied = "retention_applied"
// EventRetentionRuleCreate/Update/Delete record CRUD changes to GoBD
// retention rules (internal/storage/retention_rules.go,
// internal/api/retention_rule_handlers.go). Compliance-critical: changing a
// retention period alters how long documents must be kept, so every mutation
// — success and failure — is audit-logged.
EventRetentionRuleCreate = "retention_rule_create"
EventRetentionRuleUpdate = "retention_rule_update"
EventRetentionRuleDelete = "retention_rule_delete"
// Digitale Akten (digitaler Aktenordner — internal/storage/akten.go,
// internal/api/akte_handlers.go). Create/Update/Close/Delete cover the akte
// lifecycle; DocumentAdd/DocumentRemove record assigning/removing a document
// to/from an akte (PUT /api/documents/{id}/akte). An akte has no own ACL —
// its visibility derives from the documents it contains (see
// project_akte_konzept_plan.md).
EventAkteCreate = "akte_create"
EventAkteUpdate = "akte_update"
EventAkteClose = "akte_close"
EventAkteDelete = "akte_delete"
EventAkteDocumentAdd = "akte_document_add"
EventAkteDocumentRemove = "akte_document_remove"
// EventDocumentSplit records a barcode-separator-page split at ingest
// (internal/pagesplit, internal/api/document_handlers.go storeUploadedFile).
// GoBD-Nachvollziehbarkeit: the uploaded multi-page original is NOT archived
// as such — it is replaced by N part documents — so the split itself is the
// only record tying the parts back to the original upload. Detail therefore
// carries the original filename, its SHA-256, the page count, the separator
// page numbers and the resulting document IDs. Logged on success and on
// failure (Success:false when the split was attempted but aborted, in which
// case the original is archived unsplit).
EventDocumentSplit = "document_split"
// EventDocumentExport records a single-document export
// (GET /api/documents/{id}/export): a ZIP containing the original WORM file,
// metadata.json and, if present, ocr_text.txt. Read-only, but archived
// content plus its full metadata leaves the system as a package, so — like
// EventComplianceExport / EventAccountingPull — it is logged like a mutation,
// including failures (Success:false).
EventDocumentExport = "document_export"
// EventDocumentBulkExport records a multi-document export
// (POST /api/documents/export): one ZIP with a doc-<id>/ folder per
// document plus index.csv. Exactly ONE entry per request (not per
// document) — Detail carries "exported=<n> skipped=<m>", where skipped
// counts documents the caller could not see or that failed to read (also
// listed in the archive's errors.txt). Logged on failure too
// (Success:false), including rejected requests (too many IDs, invalid
// filter) and aborted streams.
EventDocumentBulkExport = "document_bulk_export"
)
// Compliance-Export (internal/api/compliance_handlers.go).
const (
// EventComplianceExport records a generated GoBD-Verfahrensdokumentation
// draft. Read-only, but exported tenant configuration leaves the system, so
// it is logged like a mutation (including failures) — and for a superadmin
// cross-tenant export the Detail carries the target tenant_id.
EventComplianceExport = "compliance_procedure_doc_export"
)
// Buchhaltungs-Pull-API (internal/api/accounting_handlers.go,
// internal/storage/accounting_api_keys.go).
const (
// EventAccountingKeyCreated/Revoked record the lifecycle of a per-tenant
// accounting API key. The plaintext key is NEVER part of Detail — only the
// key id and its label.
EventAccountingKeyCreated = "accounting_key_created"
EventAccountingKeyRevoked = "accounting_key_revoked"
// EventAccountingPull records machine-to-machine reads through an
// accounting API key. Read-only, but archived content leaves the system via
// a non-browser path, so it is logged like a mutation (including failures):
// every file download individually, and every list query with its result
// range (first/last document id).
EventAccountingPull = "accounting_pull"
)
// Entry is a single audit log record.
type Entry struct {
ID int64 `json:"id"`
Timestamp time.Time `json:"timestamp"`
EventType string `json:"event_type"`
Username string `json:"username"`
IPAddress string `json:"ip_address"`
DocumentID string `json:"document_id"`
Success bool `json:"success"`
Detail string `json:"detail"`
// TenantID, when set, records which tenant this event belongs to. nil means
// a tenant-less / system-wide event (e.g. superadmin actions, cron jobs).
TenantID *int64 `json:"tenant_id,omitempty"`
}
// QueryFilter specifies filtering options for audit log queries.
type QueryFilter struct {
Username string
EventType string
DocumentID string
From *time.Time
To *time.Time
TenantID *int64
PageSize int
Page int
}
// Logger is a PostgreSQL-backed, append-only audit log mirrored to a
// tamper-evident JSON-Lines file opened in append-only mode.
type Logger struct {
pool *pgxpool.Pool
logger *slog.Logger
fileMu sync.Mutex
file *os.File
logPath string
}
type fileEntry struct {
Timestamp string `json:"timestamp"`
EventType string `json:"event_type"`
Username string `json:"username"`
IPAddress string `json:"ip_address"`
DocumentID string `json:"document_id,omitempty"`
Success bool `json:"success"`
Detail string `json:"detail,omitempty"`
TenantID *int64 `json:"tenant_id,omitempty"`
}
// New connects to PostgreSQL using the given DSN and initialises the schema.
func New(dsn, logPath string, logger *slog.Logger) (*Logger, error) {
ctx := context.Background()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
return nil, fmt.Errorf("audit: connect: %w", err)
}
if err := initSchema(ctx, pool); err != nil {
pool.Close()
return nil, fmt.Errorf("audit: create schema: %w", err)
}
l := &Logger{pool: pool, logger: logger, logPath: logPath}
l.openLogFile()
return l, nil
}
// initSchema creates the audit_log table and installs the immutability
// trigger. Both operations are idempotent and safe on existing databases.
func initSchema(ctx context.Context, pool *pgxpool.Pool) error {
if _, err := pool.Exec(ctx, `
CREATE TABLE IF NOT EXISTS audit_log (
id BIGSERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
event_type VARCHAR(50) NOT NULL,
username VARCHAR(255) NOT NULL DEFAULT '',
ip_address VARCHAR(45) NOT NULL DEFAULT '',
document_id VARCHAR(64) NOT NULL DEFAULT '',
success BOOLEAN NOT NULL DEFAULT true,
detail TEXT NOT NULL DEFAULT '',
tenant_id BIGINT
);
`); err != nil {
return err
}
// Append-only enforcement at the database level: any UPDATE/DELETE raises.
if _, err := pool.Exec(ctx, `
CREATE OR REPLACE FUNCTION audit_log_no_mutation()
RETURNS trigger AS $$
BEGIN
RAISE EXCEPTION 'audit_log is append-only: % is not permitted', TG_OP
USING ERRCODE = 'integrity_constraint_violation';
END;
$$ LANGUAGE plpgsql;
`); err != nil {
return fmt.Errorf("create trigger function: %w", err)
}
if _, err := pool.Exec(ctx, `DROP TRIGGER IF EXISTS audit_log_immutable ON audit_log;`); err != nil {
return fmt.Errorf("drop trigger: %w", err)
}
if _, err := pool.Exec(ctx, `
CREATE TRIGGER audit_log_immutable
BEFORE UPDATE OR DELETE ON audit_log
FOR EACH ROW EXECUTE FUNCTION audit_log_no_mutation();
`); err != nil {
return fmt.Errorf("create trigger: %w", err)
}
return nil
}
func (l *Logger) openLogFile() {
if l.logPath == "" {
l.logger.Warn("audit: log_path not configured, file logging disabled")
return
}
if dir := filepath.Dir(l.logPath); dir != "" && dir != "." {
_ = os.MkdirAll(dir, 0o750)
}
f, err := os.OpenFile(l.logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640)
if err != nil {
l.logger.Warn("audit: audit log file not writable, continuing with DB-only logging",
"path", l.logPath, "err", err)
return
}
l.file = f
}
// Log appends an entry to the audit log. Errors are logged but not returned.
func (l *Logger) Log(entry Entry) {
ts := entry.Timestamp
if ts.IsZero() {
ts = time.Now().UTC()
}
ctx := context.Background()
_, err := l.pool.Exec(ctx,
`INSERT INTO audit_log (timestamp, event_type, username, ip_address, document_id, success, detail, tenant_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
ts.UTC(), entry.EventType, entry.Username, entry.IPAddress, entry.DocumentID,
entry.Success, entry.Detail, entry.TenantID,
)
if err != nil {
l.logger.Error("audit: insert failed", "err", err)
}
l.writeFile(entry, ts.UTC())
}
func (l *Logger) writeFile(entry Entry, ts time.Time) {
l.fileMu.Lock()
defer l.fileMu.Unlock()
if l.file == nil {
return
}
line, err := json.Marshal(fileEntry{
Timestamp: ts.Format(time.RFC3339),
EventType: entry.EventType,
Username: entry.Username,
IPAddress: entry.IPAddress,
DocumentID: entry.DocumentID,
Success: entry.Success,
Detail: entry.Detail,
TenantID: entry.TenantID,
})
if err != nil {
l.logger.Error("audit: marshal log line failed", "err", err)
return
}
if _, err := l.file.Write(append(line, '\n')); err != nil {
l.logger.Error("audit: write to log file failed", "path", l.logPath, "err", err)
}
}
// Query retrieves audit entries matching the given filter.
func (l *Logger) Query(filter QueryFilter) ([]Entry, int, error) {
pageSize := filter.PageSize
if pageSize <= 0 {
pageSize = 50
}
where, args := buildWhere(filter)
ctx := context.Background()
countSQL := "SELECT COUNT(*) FROM audit_log" + where
var total int
if err := l.pool.QueryRow(ctx, countSQL, args...).Scan(&total); err != nil {
return nil, 0, fmt.Errorf("audit: count: %w", err)
}
offset := filter.Page * pageSize
limitArg := len(args) + 1
offsetArg := len(args) + 2
querySQL := fmt.Sprintf(
"SELECT id, timestamp, event_type, username, ip_address, document_id, success, detail FROM audit_log%s ORDER BY timestamp DESC LIMIT $%d OFFSET $%d",
where, limitArg, offsetArg,
)
allArgs := append(args, pageSize, offset)
rows, err := l.pool.Query(ctx, querySQL, allArgs...)
if err != nil {
return nil, 0, fmt.Errorf("audit: query: %w", err)
}
defer rows.Close()
var entries []Entry
for rows.Next() {
var e Entry
if err := rows.Scan(&e.ID, &e.Timestamp, &e.EventType, &e.Username, &e.IPAddress, &e.DocumentID, &e.Success, &e.Detail); err != nil {
return nil, 0, fmt.Errorf("audit: scan: %w", err)
}
entries = append(entries, e)
}
return entries, total, rows.Err()
}
// Close closes the audit log file and the connection pool.
func (l *Logger) Close() error {
l.fileMu.Lock()
if l.file != nil {
_ = l.file.Sync()
_ = l.file.Close()
l.file = nil
}
l.fileMu.Unlock()
l.pool.Close()
return nil
}
func buildWhere(f QueryFilter) (string, []interface{}) {
var clauses []string
var args []interface{}
n := 1
if f.Username != "" {
clauses = append(clauses, fmt.Sprintf("username = $%d", n))
args = append(args, f.Username)
n++
}
if f.EventType != "" {
clauses = append(clauses, fmt.Sprintf("event_type = $%d", n))
args = append(args, f.EventType)
n++
}
if f.DocumentID != "" {
clauses = append(clauses, fmt.Sprintf("document_id = $%d", n))
args = append(args, f.DocumentID)
n++
}
if f.From != nil {
clauses = append(clauses, fmt.Sprintf("timestamp >= $%d", n))
args = append(args, f.From.UTC())
n++
}
if f.To != nil {
clauses = append(clauses, fmt.Sprintf("timestamp <= $%d", n))
args = append(args, f.To.UTC())
n++
}
if f.TenantID != nil {
clauses = append(clauses, fmt.Sprintf("tenant_id = $%d", n))
args = append(args, *f.TenantID)
n++
}
if len(clauses) == 0 {
return "", args
}
return " WHERE " + strings.Join(clauses, " AND "), args
}