scripte/archivdms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FDN-01: repository & projektgerüst

Browse Source 
Git-Repository für bestehenden archivdms-Code initialisiert, Branch-/Commit-Konvention (feature/<ticket>-<slug>-Branches, Ticket-Prefix in Commit-Nachricht) etabliert.
This commit is contained in:
patrick
2026-08-11 21:27:53 +02:00
parent 40ed80da71
commit 9a24ea29e1
274 changed files with 53708 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/config.go
+415
View File
@@ -0,0 +1,415 @@
// Package config loads the archivdms application configuration from a YAML
// file. The structure mirrors the archivmail config pattern (Server/Database/
// API/SMTPOut/Audit sections) but drops everything mail-specific (IMAP/POP3/
// SMTP daemon, index backend, etc.) since archivdms is a document-centric DMS.
package config
import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
"gopkg.in/yaml.v3"
)
// APIConfig holds configuration for the HTTP API server.
type APIConfig struct {
Bind string `yaml:"bind"`
// Secret is the master secret from which the JWT signing key is derived.
Secret string `yaml:"secret"`
// SecureCookies sets the Secure flag on session cookies. Enable when TLS is
// terminated at this server or at a trusted reverse proxy.
SecureCookies bool `yaml:"secure_cookies"`
// TrustedProxies is a list of IP addresses or CIDR ranges whose
// X-Forwarded-For header is trusted. Empty = trust no proxy.
TrustedProxies []string `yaml:"trusted_proxies"`
}
// ServerConfig holds general server settings.
type ServerConfig struct {
FQDN string `yaml:"fqdn"` // used for generated links (invite/reset mails)
APIPort int `yaml:"api_port"`
}
// DatabaseConfig holds PostgreSQL connection settings.
type DatabaseConfig struct {
Host string `yaml:"host"`
Port int `yaml:"port"`
Name string `yaml:"name"`
User string `yaml:"user"`
Password string `yaml:"password"`
SSLMode string `yaml:"sslmode"`
}
// DSN builds a PostgreSQL connection string from the config fields.
func (d DatabaseConfig) DSN() string {
return fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=%s",
d.User, d.Password, d.Host, d.Port, d.Name, d.SSLMode)
}
// SMTPOutConfig holds settings for outgoing transactional email (invites,
// password reset, reminder notifications).
type SMTPOutConfig struct {
Host string `yaml:"host"`
Port int `yaml:"port"`
User string `yaml:"user"`
Password string `yaml:"password"`
TLS bool `yaml:"tls"`
From string `yaml:"from"` // e.g. "archivdms <noreply@firma.de>"
}
// DefaultAuditLogPath is the default location of the append-only JSON-Lines
// audit log file when audit.log_path is not configured.
const DefaultAuditLogPath = "/var/log/archivdms/audit.log"
// AuditConfig holds audit log settings.
type AuditConfig struct {
LogPath string `yaml:"log_path"`
RetentionDays int `yaml:"retention_days"`
}
// ResolvedLogPath returns the configured audit log file path, falling back to
// DefaultAuditLogPath when unset.
func (a AuditConfig) ResolvedLogPath() string {
if strings.TrimSpace(a.LogPath) == "" {
return DefaultAuditLogPath
}
return a.LogPath
}
// LoggingConfig holds application logging settings.
type LoggingConfig struct {
Path string `yaml:"path"`
Level string `yaml:"level"`
}
// StorageConfig holds settings for document blob storage on disk.
//
// Layout under BasePath (see internal/ocr and internal/api upload handler):
//
// <BasePath>/inbox/<tenant_id>/<uuid>.<ext> raw upload, pre-processing
// <BasePath>/store/<tenant_id>/<yyyy>/<mm>/<sha256>.<ext> finished archive (WORM, 0440)
// <BasePath>/ocr-tmp/<uuid>/ pdftoppm scratch, removed after use
type StorageConfig struct {
// BasePath is the root directory for inbox/store/ocr-tmp (see helper
// methods below). Replaces the old, unused StorePath/store_path field.
BasePath string `yaml:"base_path"`
// RetentionDays is the default GoBD retention period (days) applied to new
// documents when no explicit retain_until is given. 0 = no default lock.
RetentionDays int `yaml:"retention_days"`
// MaxUploadSizeMB caps the accepted multipart upload size. 0 = default 50.
MaxUploadSizeMB int `yaml:"max_upload_size_mb"`
}
// InboxPath returns the directory raw uploads are written to before hashing
// and OCR processing.
func (s StorageConfig) InboxPath() string { return filepath.Join(s.BasePath, "inbox") }
// StorePath returns the directory the finished, content-addressed WORM
// archive lives in.
func (s StorageConfig) StorePath() string { return filepath.Join(s.BasePath, "store") }
// OCRTmpPath returns the scratch directory for pdftoppm intermediate images.
func (s StorageConfig) OCRTmpPath() string { return filepath.Join(s.BasePath, "ocr-tmp") }
// ThumbnailPath returns the directory holding derived preview thumbnails.
// Thumbnails are regenerable artefacts (not WORM): they may be deleted at any
// time and are re-created lazily on next request. Layout mirrors the store:
// thumbnails/<tenant_id>/<content_hash>.png.
func (s StorageConfig) ThumbnailPath() string { return filepath.Join(s.BasePath, "thumbnails") }
// ResolvedMaxUploadSizeMB returns MaxUploadSizeMB, falling back to a default
// of 50 MB when unset (<= 0).
func (s StorageConfig) ResolvedMaxUploadSizeMB() int {
if s.MaxUploadSizeMB <= 0 {
return 50
}
return s.MaxUploadSizeMB
}
// OCRConfig holds settings for the tesseract/poppler-utils OCR sidecar
// pipeline (internal/ocr). All binaries are optional system packages — a
// missing binary degrades OCR to a no-op rather than failing the upload.
type OCRConfig struct {
// TesseractPath is the path/name of the tesseract binary. Default "tesseract".
TesseractPath string `yaml:"tesseract_path"`
// PdftoppmPath is the path/name of the pdftoppm binary. Default "pdftoppm".
PdftoppmPath string `yaml:"pdftoppm_path"`
// Languages is the tesseract -l argument, e.g. "deu+eng". Default "deu+eng".
Languages string `yaml:"languages"`
// TimeoutSeconds bounds each individual OCR subprocess call. Default 60.
TimeoutSeconds int `yaml:"timeout_seconds"`
// SofficePath is the path/name of the LibreOffice headless binary used to
// convert Office documents (docx/xlsx/pptx/odt/...) to PDF before OCR.
// Default "soffice". A missing binary degrades Office ingest to a no-op.
SofficePath string `yaml:"soffice_path"`
// BinarizeOCR enables an Otsu auto-threshold (black/white) step at the end
// of the image preprocessing pipeline (internal/ocr.Extractor.Binarize),
// after deskew/contrast-normalize/OSD-rotation. Default false: this is a
// lossy step (every pixel becomes pure black or white) that helps flat
// text scans but can hurt documents with color stamps/signatures or
// embedded photos — enable per-deployment only after validating against
// that corpus. Requires ImageMagick 7 (`-auto-threshold` syntax); a
// missing/older convert binary just skips the step (best-effort, same as
// every other preprocessing step here).
BinarizeOCR bool `yaml:"binarize_ocr"`
// DeskewMethod selects the fine-skew-angle correction strategy used in
// internal/ocr.Extractor.runTesseract before the OSD 90-degree rotation
// pass. "imagemagick" (default/empty) keeps the existing behavior:
// ImageMagick's own `-deskew 40%` peak/valley background-projection
// analysis (internal/ocr.deskewImage). "hough" instead detects the angle
// via a Python/OpenCV sidecar script (internal/ocr/scripts/
// hough_deskew.py, minAreaRect/HoughLinesP-based) and applies it with a
// plain `convert -rotate <deg>` — separating angle detection from angle
// application, which the ImageMagick approach does not do and which
// fails on tightly-cropped phone photos lacking background margin.
// Requires python3 + opencv-python (Debian: python3-opencv) on the host;
// falls back to no-op (angle 0, same as any other best-effort
// preprocessing step here) if the script/dependency is missing.
DeskewMethod string `yaml:"deskew_method"`
// HoughDeskewScriptPath overrides the path to hough_deskew.py. Empty
// defaults to "/opt/archivdms/scripts/hough_deskew.py" (the on-premise
// install layout — see install.sh/update.sh, INSTALL_DIR=/opt/archivdms).
// Only consulted when DeskewMethod == "hough".
HoughDeskewScriptPath string `yaml:"hough_deskew_script_path"`
}
// ResolvedTesseractPath returns TesseractPath, defaulting to "tesseract".
func (o OCRConfig) ResolvedTesseractPath() string {
if strings.TrimSpace(o.TesseractPath) == "" {
return "tesseract"
}
return o.TesseractPath
}
// ResolvedPdftoppmPath returns PdftoppmPath, defaulting to "pdftoppm".
func (o OCRConfig) ResolvedPdftoppmPath() string {
if strings.TrimSpace(o.PdftoppmPath) == "" {
return "pdftoppm"
}
return o.PdftoppmPath
}
// ResolvedSofficePath returns SofficePath, defaulting to "soffice".
func (o OCRConfig) ResolvedSofficePath() string {
if strings.TrimSpace(o.SofficePath) == "" {
return "soffice"
}
return o.SofficePath
}
// ResolvedDeskewMethod returns DeskewMethod, defaulting to "imagemagick"
// (the pre-existing behavior — see the DeskewMethod field doc). Any value
// other than "hough" is treated as "imagemagick" so a typo in config.yaml
// degrades to the known-safe default rather than silently disabling deskew.
func (o OCRConfig) ResolvedDeskewMethod() string {
if strings.TrimSpace(strings.ToLower(o.DeskewMethod)) == "hough" {
return "hough"
}
return "imagemagick"
}
// ResolvedHoughDeskewScriptPath returns HoughDeskewScriptPath, defaulting to
// "/opt/archivdms/scripts/hough_deskew.py" (see the field doc).
func (o OCRConfig) ResolvedHoughDeskewScriptPath() string {
if strings.TrimSpace(o.HoughDeskewScriptPath) == "" {
return "/opt/archivdms/scripts/hough_deskew.py"
}
return o.HoughDeskewScriptPath
}
// ResolvedLanguages returns Languages, defaulting to "deu+eng".
func (o OCRConfig) ResolvedLanguages() string {
if strings.TrimSpace(o.Languages) == "" {
return "deu+eng"
}
return o.Languages
}
// ResolvedTimeout returns TimeoutSeconds as a time.Duration, defaulting to 60s.
func (o OCRConfig) ResolvedTimeout() time.Duration {
if o.TimeoutSeconds <= 0 {
return 60 * time.Second
}
return time.Duration(o.TimeoutSeconds) * time.Second
}
// SFTPConfig holds settings for the embedded per-tenant SFTP server
// (internal/sftpserver). Disabled by default — no port is opened unless
// explicitly enabled.
type SFTPConfig struct {
// Enabled turns the embedded SFTP server on/off. Default false.
Enabled bool `yaml:"enabled"`
// Bind is the listen address, e.g. ":2222".
Bind string `yaml:"bind"`
// HostKeyPath is where the server's SSH host key is persisted. Generated
// on first start if missing. Defaults to "<storage.base_path>/.ssh/host_key".
HostKeyPath string `yaml:"host_key_path"`
}
// ResolvedBind returns Bind, defaulting to ":2222".
func (c SFTPConfig) ResolvedBind() string {
if strings.TrimSpace(c.Bind) == "" {
return ":2222"
}
return c.Bind
}
// ResolvedHostKeyPath returns HostKeyPath, defaulting to
// "<basePath>/.ssh/host_key" when unset.
func (c SFTPConfig) ResolvedHostKeyPath(basePath string) string {
if strings.TrimSpace(c.HostKeyPath) != "" {
return c.HostKeyPath
}
return filepath.Join(basePath, ".ssh", "host_key")
}
// IndexConfig holds settings for the optional full-text search index
// (internal/index, Manticore Search over the MySQL protocol, port 9306).
//
// Phase 1 wires only the write/sync layer — there is no search endpoint yet.
// When ManticoreDSN is empty the index is disabled entirely: the store's
// Indexer stays nil and every sync call is a silent no-op (Postgres remains the
// single source of truth).
type IndexConfig struct {
// ManticoreDSN is a go-sql-driver/mysql DSN pointing at Manticore's SQL
// port, e.g. "archivdms@tcp(127.0.0.1:9306)/?charset=utf8mb4". Empty =
// index disabled.
ManticoreDSN string `yaml:"manticore_dsn"`
}
// JobQueueConfig holds settings for the tenant-fair, Postgres-backed
// processing queue (internal/jobqueue): OCR extraction, taxonomy
// auto-assignment and on_upload workflows run asynchronously in worker
// goroutines inside this same process (no separate service/container).
//
// All fields are optional — the defaults below are tuned for a single
// mid-sized server running Tesseract locally.
type JobQueueConfig struct {
// Disabled turns the async pipeline off entirely. Documents then stay in
// processing_status='queued' until the queue is enabled again (nothing is
// lost — the WORM file and the job row are already persisted). Default
// false, i.e. the queue runs.
Disabled bool `yaml:"disabled"`
// Workers is the number of concurrent worker goroutines. Default 2 —
// Tesseract is CPU-bound, more workers than cores hurts.
Workers int `yaml:"workers"`
// PollIntervalMS is how often the dispatcher looks for due jobs.
// Default 2000 (2s).
PollIntervalMS int `yaml:"poll_interval_ms"`
// JobTimeoutSeconds bounds one job run and doubles as the reaper's
// threshold for stuck 'processing' rows. Default 600 (10 min) — large
// multi-page scans plus LibreOffice conversion can legitimately take
// minutes.
JobTimeoutSeconds int `yaml:"job_timeout_seconds"`
// MaxRetries is the retry_count cap. Once exceeded, the job stays
// permanently 'failed' and is only retried on explicit manual request.
// Default 5.
MaxRetries int `yaml:"max_retries"`
}
// ResolvedWorkers returns Workers, defaulting to 2.
func (j JobQueueConfig) ResolvedWorkers() int {
if j.Workers <= 0 {
return 2
}
return j.Workers
}
// ResolvedPollInterval returns PollIntervalMS as a Duration, default 2s.
func (j JobQueueConfig) ResolvedPollInterval() time.Duration {
if j.PollIntervalMS <= 0 {
return 2 * time.Second
}
return time.Duration(j.PollIntervalMS) * time.Millisecond
}
// ResolvedJobTimeout returns JobTimeoutSeconds as a Duration, default 10min.
func (j JobQueueConfig) ResolvedJobTimeout() time.Duration {
if j.JobTimeoutSeconds <= 0 {
return 10 * time.Minute
}
return time.Duration(j.JobTimeoutSeconds) * time.Second
}
// ResolvedMaxRetries returns MaxRetries, defaulting to 5.
func (j JobQueueConfig) ResolvedMaxRetries() int {
if j.MaxRetries <= 0 {
return 5
}
return j.MaxRetries
}
// PageSplitConfig holds settings for barcode separator-page splitting at
// ingest (internal/pagesplit). A printed separator sheet carrying the
// configured barcode cuts a multi-page PDF scan into several individual
// documents; the separator page itself is discarded.
//
// Deliberately disabled by default and configured globally (config.yaml)
// rather than per tenant for now: this is the first iteration, and the
// project rule for new preprocessing behaviour is to introduce it
// conservatively and validate it in the field before turning it on broadly
// (same approach as ocr.binarize_ocr). A per-tenant setting plus a settings UI
// is the intended next step.
type PageSplitConfig struct {
// Enabled turns separator-page splitting on. Default false.
Enabled bool `yaml:"enabled"`
// Marker is the barcode payload identifying a separator page. Empty
// defaults to pagesplit.DefaultMarker ("ARCHIVDMS-SPLIT"). Matched
// case-insensitively after trimming.
Marker string `yaml:"marker"`
// MarkerPrefix switches matching from equality to prefix matching, so a
// separator sheet may carry additional payload after the marker.
MarkerPrefix bool `yaml:"marker_prefix"`
// RasterDPI is the resolution separator detection rasterizes pages at.
// 0 = default 150 (enough for a full-page separator barcode, far cheaper
// than the 300 dpi OCR pass).
RasterDPI int `yaml:"raster_dpi"`
// MaxPages caps how many pages are analysed; longer documents are archived
// unsplit. 0 = default 200.
MaxPages int `yaml:"max_pages"`
// TimeoutSeconds bounds each poppler subprocess call. 0 = default 120.
TimeoutSeconds int `yaml:"timeout_seconds"`
}
// Config is the full application configuration loaded from YAML.
type Config struct {
Server ServerConfig `yaml:"server"`
Storage StorageConfig `yaml:"storage"`
OCR OCRConfig `yaml:"ocr"`
PageSplit PageSplitConfig `yaml:"pagesplit"`
JobQueue JobQueueConfig `yaml:"jobqueue"`
SFTP SFTPConfig `yaml:"sftp"`
Index IndexConfig `yaml:"index"`
Database DatabaseConfig `yaml:"database"`
SMTPOut SMTPOutConfig `yaml:"smtp_out"`
API APIConfig `yaml:"api"`
Audit AuditConfig `yaml:"audit"`
Logging LoggingConfig `yaml:"logging"`
}
// Load reads a YAML config file from path and returns a parsed Config.
// It also bootstraps the storage directory tree (inbox/store/ocr-tmp) so the
// upload handler and OCR pipeline can rely on it existing at startup.
func Load(path string) (*Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var cfg Config
if err := yaml.Unmarshal(data, &cfg); err != nil {
return nil, err
}
if cfg.Storage.BasePath != "" {
for _, dir := range []string{cfg.Storage.InboxPath(), cfg.Storage.StorePath(), cfg.Storage.OCRTmpPath(), cfg.Storage.ThumbnailPath()} {
if err := os.MkdirAll(dir, 0o750); err != nil {
return nil, fmt.Errorf("config: create storage dir %s: %w", dir, err)
}
}
}
return &cfg, nil
}
config/config.yml.example
+80
View File
@@ -0,0 +1,80 @@
server:
fqdn: "dms.example.com"
api_port: 8080
storage:
base_path: "/var/lib/archivdms" # enthält inbox/, store/, ocr-tmp/ (siehe README)
retention_days: 3650 # GoBD default: 10 Jahre
max_upload_size_mb: 50
ocr:
tesseract_path: "tesseract" # muss im PATH liegen, apt install tesseract-ocr tesseract-ocr-deu
pdftoppm_path: "pdftoppm" # apt install poppler-utils
languages: "deu+eng"
timeout_seconds: 60
pagesplit:
# Trennseiten-Split (internal/pagesplit): mehrseitige PDF-Scans werden beim
# Ingest an Barcode-Trennblättern in Einzeldokumente zerlegt. Das Trennblatt
# selbst wird verworfen, jedes Teildokument durchläuft den normalen Pfad
# (eigene WORM-Ablage, eigene OCR, eigene Autozuordnung). Der Split wird im
# Audit-Log als "document_split" mit Seitenbereichen + Dokument-IDs
# protokolliert (GoBD-Nachvollziehbarkeit).
# Braucht: apt install poppler-utils zbar-tools
enabled: false # Default AUS — erst nach Feldvalidierung aktivieren
marker: "ARCHIVDMS-SPLIT" # leer = Default "ARCHIVDMS-SPLIT" (Vergleich case-insensitiv)
marker_prefix: false # true = Trennblatt-Barcode muss nur mit marker BEGINNEN
raster_dpi: 150 # 0 = Default 150 (reicht für einen ganzseitigen Barcode)
max_pages: 200 # 0 = Default 200; längere PDFs werden ungesplittet archiviert
timeout_seconds: 120 # 0 = Default 120, je poppler-Aufruf
jobqueue:
# Mandanten-faire Verarbeitungs-Queue (internal/jobqueue): OCR, Taxonomie-
# Autozuordnung und on_upload-Workflows laufen asynchron in Worker-Goroutinen
# im selben Prozess (kein separater Dienst, kein Redis). Der Upload-Request
# macht nur noch Datei speichern + WORM + Job anlegen.
disabled: false # true = keine Nachverarbeitung (Dokumente bleiben "queued")
workers: 2 # Tesseract ist CPU-gebunden — nicht über die Kernzahl gehen
poll_interval_ms: 2000 # wie oft der Dispatcher nach fälligen Jobs schaut
job_timeout_seconds: 600 # Limit je Job, zugleich Schwelle des Reapers für hängende Jobs
max_retries: 5 # danach dauerhaft "failed", kein Automatik-Retry mehr
sftp:
enabled: false # eingebetteter SFTP-Server pro Mandant (internal/sftpserver)
bind: ":2222"
host_key_path: "" # leer = <storage.base_path>/.ssh/host_key (wird beim ersten Start generiert)
index:
# Volltext-Index (Manticore, MySQL-Protokoll Port 9306). Leer = deaktiviert.
# Phase 1: nur Schreib-/Sync-Layer, noch KEIN Such-Endpunkt.
manticore_dsn: "" # z.B. "archivdms@tcp(127.0.0.1:9306)/?charset=utf8mb4"
database:
host: "127.0.0.1"
port: 5432
name: "archivdms"
user: "archivdms"
password: "CHANGE_ME"
sslmode: "disable"
smtp_out:
host: "smtp.example.com"
port: 587
user: "noreply@example.com"
password: "CHANGE_ME"
tls: false
from: "archivdms <noreply@example.com>"
api:
bind: ":8080"
secret: "CHANGE_ME_TO_A_LONG_RANDOM_SECRET"
secure_cookies: true
trusted_proxies: []
audit:
log_path: "/var/log/archivdms/audit.log"
retention_days: 3650
logging:
path: "/var/log/archivdms/app.log"
level: "info"