FDN-02/FDN-03/FDN-07/FDN-08: Migrations-Rollback, Objekt-Storage-Interface, go.sum-Fix, Observability
CI / Backend (go vet, go test -cover) (push) Has been cancelled
CI / Frontend (ESLint, tsc, next build) (push) Has been cancelled

- FDN-02: Rollback-fähige Down-Migrationen (024-026), archivdms seed dev CLI
- FDN-03: internal/objectstore Interface + lokaler WORM-Treiber, signierte Download-URLs
- FDN-07: go.mod/go.sum vervollständigt (fehlender go-ldap/v3-Eintrag), CI-Pipeline (.gitea/workflows/ci.yml, bereits in FDN-01 committet) damit lauffähig
- FDN-08: Request-ID-Middleware, /metrics-Endpoint, Panic-Recovery, Login/Logout/Me technisches Logging inkl. Access-Log je Anfrage
This commit is contained in:
2026-08-11 22:27:52 +02:00
parent 9a24ea29e1
commit 89de794356
30 changed files with 2688 additions and 113 deletions
+4 -4
View File
@@ -190,7 +190,7 @@ func (s *Server) accountingAuth(h http.HandlerFunc) http.HandlerFunc {
tenantID, keyID, err := s.store.ResolveAccountingAPIKey(r.Context(), rawKey)
if err != nil {
if !errors.Is(err, storage.ErrAccountingKeyNotFound) {
s.logger.Error("accounting api key resolve failed", "err", err)
s.reqLog(r.Context()).Error("accounting api key resolve failed", "err", err)
}
// Unknown, revoked and broken keys are indistinguishable.
s.audlog.Log(audit.Entry{
@@ -290,7 +290,7 @@ func (s *Server) handleAccountingListDocuments(w http.ResponseWriter, r *http.Re
writeError(w, http.StatusBadRequest, "invalid cursor")
return
}
s.logger.Error("accounting list failed", "tenant_id", tenantID, "err", err)
s.reqLog(r.Context()).Error("accounting list failed", "tenant_id", tenantID, "err", err)
s.audlog.Log(audit.Entry{
EventType: audit.EventAccountingPull, Username: "accounting_api", IPAddress: s.remoteIP(r),
TenantID: &tenantID, Success: false,
@@ -342,7 +342,7 @@ func (s *Server) handleAccountingDocumentFile(w http.ResponseWriter, r *http.Req
f, err := os.Open(ref.StoragePath())
if err != nil {
s.logger.Error("accounting file open failed", "document_id", ref.DocumentID, "tenant_id", tenantID, "err", err)
s.reqLog(r.Context()).Error("accounting file open failed", "document_id", ref.DocumentID, "tenant_id", tenantID, "err", err)
s.audlog.Log(audit.Entry{
EventType: audit.EventAccountingPull, Username: "accounting_api", IPAddress: s.remoteIP(r),
TenantID: &tenantID, DocumentID: strconv.FormatInt(id, 10), Success: false,
@@ -364,7 +364,7 @@ func (s *Server) handleAccountingDocumentFile(w http.ResponseWriter, r *http.Req
w.Header().Set("Content-Disposition", "attachment; filename=\""+safeDownloadName(ref.Title, ext)+"\"")
w.Header().Set("X-Content-Type-Options", "nosniff")
if _, err := io.Copy(w, f); err != nil {
s.logger.Warn("accounting file stream interrupted", "document_id", ref.DocumentID, "err", err)
s.reqLog(r.Context()).Warn("accounting file stream interrupted", "document_id", ref.DocumentID, "err", err)
}
}
+30 -5
View File
@@ -13,15 +13,22 @@ type loginRequest struct {
}
func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var req loginRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
s.reqLog(ctx).Warn("login: invalid request body",
"remote_ip", s.remoteIP(r), "err", err)
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
ip := s.remoteIP(r)
token, user, err := s.authMgr.LoginFrom(r.Context(), req.Username, req.Password, ip)
token, user, err := s.authMgr.LoginFrom(ctx, req.Username, req.Password, ip)
if err != nil {
// Kein Passwort, keine Fehlerdetails vom Auth-Manager im Klartext:
// nur Benutzername + IP zur Korrelation von Brute-Force-Versuchen.
s.reqLog(ctx).Warn("login failed",
"username", req.Username, "remote_ip", ip, "reason", "invalid_credentials")
s.audlog.Log(audit.Entry{
EventType: audit.EventLogin,
Username: req.Username,
@@ -43,7 +50,14 @@ func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
MaxAge: 8 * 60 * 60,
})
_ = s.users.UpdateLastLogin(user.ID)
if err := s.users.UpdateLastLogin(user.ID); err != nil {
s.reqLog(ctx).Warn("login: last_login update failed",
"user_id", user.ID, "err", err)
}
s.reqLog(ctx).Info("login succeeded",
"user_id", user.ID, "username", user.Username,
"tenant_id", user.TenantID, "remote_ip", ip)
s.audlog.Log(audit.Entry{
EventType: audit.EventLogin,
@@ -57,9 +71,12 @@ func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) handleMe(w http.ResponseWriter, r *http.Request) {
sess := sessionFromCtx(r.Context())
ctx := r.Context()
sess := sessionFromCtx(ctx)
user, err := s.users.GetByID(sess.UserID)
if err != nil {
s.reqLog(ctx).Warn("me: user lookup failed",
"user_id", sess.UserID, "err", err)
writeError(w, http.StatusNotFound, "user not found")
return
}
@@ -67,7 +84,8 @@ func (s *Server) handleMe(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request) {
sess := sessionFromCtx(r.Context())
ctx := r.Context()
sess := sessionFromCtx(ctx)
token := ""
if c, err := r.Cookie(sessionCookieName); err == nil {
token = c.Value
@@ -76,9 +94,16 @@ func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request) {
token = extractBearerToken(r)
}
if token != "" {
_ = s.authMgr.Logout(token)
if err := s.authMgr.Logout(token); err != nil {
s.reqLog(ctx).Warn("logout: session invalidation failed",
"user_id", sess.UserID, "err", err)
}
}
s.reqLog(ctx).Info("logout",
"user_id", sess.UserID, "username", sess.Username,
"tenant_id", sess.TenantID, "remote_ip", s.remoteIP(r))
http.SetCookie(w, &http.Cookie{
Name: sessionCookieName,
Value: "",
+1 -1
View File
@@ -21,7 +21,7 @@ func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) {
stats, err := s.store.GetDashboardStats(r.Context(), *sess.TenantID, sess.UserID)
if err != nil {
s.logger.Error("dashboard stats failed", "err", err)
s.reqLog(r.Context()).Error("dashboard stats failed", "err", err)
writeError(w, http.StatusInternalServerError, "dashboard stats failed")
return
}
@@ -197,7 +197,7 @@ func (s *Server) handleBulkExportDocuments(w http.ResponseWriter, r *http.Reques
if err != nil {
// A single unreadable document must not kill the archive — unless the
// ZIP writer itself failed, which we detect on Close below.
s.logger.Warn("bulk export: document skipped", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(r.Context()).Warn("bulk export: document skipped", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
skipped = append(skipped, fmt.Sprintf("%d: %v", doc.ID, err))
continue
}
@@ -225,7 +225,7 @@ func (s *Server) handleBulkExportDocuments(w http.ResponseWriter, r *http.Reques
detail := fmt.Sprintf("zip_bulk_export: exported=%d skipped=%d", exported, len(skipped))
if streamErr != nil {
// Headers are already out — audit the partial export, no HTTP error.
s.logger.Warn("bulk document export stream failed", "tenant_id", tenantID, "err", streamErr)
s.reqLog(r.Context()).Warn("bulk document export stream failed", "tenant_id", tenantID, "err", streamErr)
s.audlog.Log(audit.Entry{
EventType: audit.EventDocumentBulkExport, Username: sess.Username, TenantID: sess.TenantID,
Success: false, Detail: detail + " stream_failed: " + streamErr.Error(),
+2 -2
View File
@@ -143,7 +143,7 @@ func (s *Server) handleExportDocument(w http.ResponseWriter, r *http.Request) {
f, err := os.Open(doc.StoragePath)
if err != nil {
s.logger.Error("export: document file open failed", "document_id", doc.ID, "tenant_id", *sess.TenantID, "err", err)
s.reqLog(r.Context()).Error("export: document file open failed", "document_id", doc.ID, "tenant_id", *sess.TenantID, "err", err)
fail(http.StatusInternalServerError, "file unavailable", "file_open_failed: "+err.Error())
return
}
@@ -212,7 +212,7 @@ func (s *Server) handleExportDocument(w http.ResponseWriter, r *http.Request) {
if streamErr != nil {
// Headers are already out — log + audit the partial export, no HTTP error.
s.logger.Warn("document export stream failed", "document_id", doc.ID, "err", streamErr)
s.reqLog(r.Context()).Warn("document export stream failed", "document_id", doc.ID, "err", streamErr)
s.audlog.Log(audit.Entry{
EventType: audit.EventDocumentExport, Username: sess.Username, TenantID: sess.TenantID,
DocumentID: idStr, Success: false, Detail: "stream_failed: " + streamErr.Error(),
+43 -84
View File
@@ -21,6 +21,7 @@ import (
"archivdms/internal/auth"
"archivdms/internal/dateformat"
"archivdms/internal/matching"
"archivdms/internal/objectstore"
"archivdms/internal/ocr"
"archivdms/internal/storage"
"archivdms/internal/userstore"
@@ -138,10 +139,12 @@ func (s *Server) handleGetDocumentFile(w http.ResponseWriter, r *http.Request) {
return
}
f, err := os.Open(doc.StoragePath)
// Read through the storage abstraction: it re-checks that the stored path
// really lies inside this tenant's store subtree before opening.
f, err := s.objects.Open(r.Context(), *sess.TenantID, doc.StoragePath)
if err != nil {
// WORM store should always hold the file, but never trust the disk.
s.logger.Error("document file open failed", "document_id", doc.ID, "tenant_id", *sess.TenantID, "storage_path", doc.StoragePath, "err", err)
s.reqLog(r.Context()).Error("document file open failed", "document_id", doc.ID, "tenant_id", *sess.TenantID, "storage_path", doc.StoragePath, "err", err)
writeError(w, http.StatusInternalServerError, "file unavailable")
return
}
@@ -152,7 +155,7 @@ func (s *Server) handleGetDocumentFile(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Disposition", "inline; filename=\""+safeDownloadName(doc.Title, ext)+"\"")
w.Header().Set("X-Content-Type-Options", "nosniff")
if _, err := io.Copy(w, f); err != nil {
s.logger.Warn("document file stream interrupted", "document_id", doc.ID, "err", err)
s.reqLog(r.Context()).Warn("document file stream interrupted", "document_id", doc.ID, "err", err)
}
}
@@ -209,7 +212,7 @@ func (s *Server) handleGetDocumentThumbnail(w http.ResponseWriter, r *http.Reque
w.Header().Set("Cache-Control", "private, max-age=86400")
w.Header().Set("X-Content-Type-Options", "nosniff")
if _, err := io.Copy(w, f); err != nil {
s.logger.Warn("thumbnail stream interrupted", "document_id", doc.ID, "err", err)
s.reqLog(r.Context()).Warn("thumbnail stream interrupted", "document_id", doc.ID, "err", err)
}
}
@@ -453,7 +456,7 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
mimeType := detectMimeType("", ext, doc.StoragePath)
result, err := s.ocr.Extract(ctx, doc.StoragePath, mimeType)
if err != nil {
s.logger.Warn("reprocess ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID, "storage_path", doc.StoragePath, "err", err)
s.reqLog(ctx).Warn("reprocess ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID, "storage_path", doc.StoragePath, "err", err)
s.audlog.Log(audit.Entry{EventType: audit.EventDocumentReprocessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "ocr_failed: " + err.Error()})
return nil, fmt.Errorf("reprocess ocr extract: %w", err)
}
@@ -466,7 +469,7 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
// alongside the new ones — best-effort, never fails the reprocess since
// ocr_text is already the authoritative persisted result.
if err := s.store.ReplaceOCRWords(ctx, id, ocrWordsFromResult(id, result.Words)); err != nil {
s.logger.Warn("reprocess replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("reprocess replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
}
if err := s.store.UpdateDocumentOCRText(ctx, id, tenantID, ocrText); err != nil {
@@ -491,7 +494,7 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
prefix, dateLayout := s.tenantScanTitleParams(ctx, tenantID)
if newTitle := titleFromOCRText(ocrText, prefix, dateLayout); newTitle != "" && newTitle != doc.Title {
if err := s.store.UpdateDocumentTitleAuto(ctx, id, tenantID, newTitle); err != nil {
s.logger.Warn("reprocess title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("reprocess title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
} else {
doc.Title = newTitle
}
@@ -513,7 +516,7 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
}
if !sameDate(datePtr, doc.DocumentDate) {
if err := s.store.UpdateDocumentDate(ctx, id, tenantID, datePtr, scorePtr); err != nil {
s.logger.Warn("reprocess document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("reprocess document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
} else {
doc.DocumentDate = datePtr
doc.DocumentDateScore = scorePtr
@@ -524,12 +527,12 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
// Additive auto-assignment (only fills unset doc_type/correspondent, only
// attaches tags) — best-effort, never fails the request.
if assignWarn := s.autoAssignTaxonomy(ctx, tenantID, doc, barcodes, ocrText); assignWarn != "" {
s.logger.Info("reprocess auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn)
s.reqLog(ctx).Info("reprocess auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn)
}
// Re-evaluate on_upload workflows — best-effort, never fails the request.
if err := s.store.RunWorkflowsForDocument(ctx, tenantID, doc, storage.WorkflowTriggerOnUpload); err != nil {
s.logger.Warn("reprocess workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("reprocess workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
}
// Re-read so the response reflects any doc_type/correspondent set by
@@ -726,11 +729,11 @@ func (s *Server) generateThumbnailBestEffort(ctx context.Context, tenantID, docu
}
thumbPath := filepath.Join(s.storageCfg.ThumbnailPath(), strconv.FormatInt(tenantID, 10), contentHash+".png")
if err := s.thumbs.Generate(ctx, storagePath, mimeType, thumbPath); err != nil {
s.logger.Warn("eager thumbnail generation failed", "document_id", documentID, "tenant_id", tenantID, "mime_type", mimeType, "err", err)
s.reqLog(ctx).Warn("eager thumbnail generation failed", "document_id", documentID, "tenant_id", tenantID, "mime_type", mimeType, "err", err)
return
}
if err := s.store.SetDocumentHasThumbnail(ctx, documentID, tenantID, true); err != nil {
s.logger.Warn("has_thumbnail flag update failed", "document_id", documentID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("has_thumbnail flag update failed", "document_id", documentID, "tenant_id", tenantID, "err", err)
}
}
@@ -753,7 +756,7 @@ func (s *Server) handleUploadDocument(w http.ResponseWriter, r *http.Request) {
tenantID := *sess.TenantID
maxBytes := int64(s.storageCfg.ResolvedMaxUploadSizeMB()) * 1024 * 1024
s.logger.Info("upload request received", "username", sess.Username, "tenant_id", tenantID,
s.reqLog(r.Context()).Info("upload request received", "username", sess.Username, "tenant_id", tenantID,
"content_length", r.ContentLength, "max_bytes", maxBytes, "remote_addr", r.RemoteAddr)
r.Body = http.MaxBytesReader(w, r.Body, maxBytes)
@@ -763,7 +766,7 @@ func (s *Server) handleUploadDocument(w http.ResponseWriter, r *http.Request) {
// hitting MaxBytesReader — both close the connection before any
// later log/audit call would otherwise run, which previously left
// zero trace of the failure in the backend logs (see DEVLOG 2026-07-15).
s.logger.Warn("upload parse failed", "username", sess.Username, "tenant_id", tenantID,
s.reqLog(r.Context()).Warn("upload parse failed", "username", sess.Username, "tenant_id", tenantID,
"content_length", r.ContentLength, "max_bytes", maxBytes, "err", err)
s.audlog.Log(audit.Entry{
EventType: audit.EventDocumentCreate, Username: sess.Username, TenantID: sess.TenantID,
@@ -956,41 +959,19 @@ func (s *Server) archiveStagedFile(
}
pathDate := time.Now()
// 4. Build the WORM target path store/<tenant>/<yyyy>/<mm>/<hash>.<ext>.
storeDir := filepath.Join(s.storageCfg.StorePath(), strconv.FormatInt(tenantID, 10),
fmt.Sprintf("%04d", pathDate.Year()), fmt.Sprintf("%02d", pathDate.Month()))
if err := os.MkdirAll(storeDir, 0o750); err != nil {
os.Remove(inboxPath)
return nil, "", fmt.Errorf("create store dir: %w", err)
}
storePath := filepath.Join(storeDir, contentHash+ext)
// 5. Collision check: identical hash already stored -> reject as
// duplicate before touching the DB (filesystem-level half of the
// duplicate protection; the DB unique index is the other half).
if _, err := os.Stat(storePath); err == nil {
os.Remove(inboxPath)
return nil, "", storage.ErrDuplicateContentHash
} else if !os.IsNotExist(err) {
os.Remove(inboxPath)
return nil, "", fmt.Errorf("stat store path: %w", err)
}
// 6. Move inbox -> store. Prefer atomic rename; fall back to copy+remove
// if inbox/store ever end up on different filesystems/mounts.
if err := os.Rename(inboxPath, storePath); err != nil {
if copyErr := copyFile(inboxPath, storePath); copyErr != nil {
os.Remove(inboxPath)
return nil, "", fmt.Errorf("move file to store: rename failed (%v), copy fallback failed: %w", err, copyErr)
// 4./5./6./7. WORM archival — delegated unchanged to the storage
// abstraction (internal/objectstore): build the target path
// store/<tenant>/<yyyy>/<mm>/<hash>.<ext>, reject an already-stored hash as
// duplicate (filesystem half of the duplicate protection; the DB unique
// index is the other half), move inbox -> store (rename, copy+remove
// fallback across mounts) and finally chmod 0440. Path scheme and
// semantics are identical to the previous inline implementation.
storePath, err := s.objects.Archive(ctx, tenantID, inboxPath, ext, contentHash, pathDate)
if err != nil {
if errors.Is(err, objectstore.ErrObjectExists) {
return nil, "", storage.ErrDuplicateContentHash
}
os.Remove(inboxPath)
}
// 7. WORM lock: read-only, no write access for anyone once archived. This is
// the ONLY chmod and it happens exactly once, after the file reaches its
// final path — the file is never moved or renamed again afterwards.
if err := os.Chmod(storePath, 0o440); err != nil {
return nil, "", fmt.Errorf("chmod store file: %w", err)
return nil, "", err
}
// 8. Record the document AND its processing job in ONE transaction. If no
@@ -1018,7 +999,7 @@ func (s *Server) archiveStagedFile(
if err != nil {
return nil, "", err
}
s.logger.Info("document staged, processing job queued",
s.reqLog(ctx).Info("document staged, processing job queued",
"document_id", doc.ID, "tenant_id", tenantID, "job_id", job.ID, "derive_title", deriveTitle)
// 7b. Eager thumbnail generation: same pipeline/cache path as the lazy
@@ -1084,7 +1065,7 @@ func (s *Server) trySplitStagedUpload(
res, split, err := s.pagesplitter.Split(ctx, inboxPath)
if err != nil {
s.logger.Warn("separator-page split failed, archiving document unsplit",
s.reqLog(ctx).Warn("separator-page split failed, archiving document unsplit",
"tenant_id", tenantID, "filename", filename, "err", err)
s.audlog.Log(audit.Entry{
EventType: audit.EventDocumentSplit, Username: splitActor, TenantID: &tenantID, Success: false,
@@ -1167,7 +1148,7 @@ func (s *Server) trySplitStagedUpload(
EventType: audit.EventDocumentSplit, Username: splitActor, TenantID: &tenantID,
DocumentID: strconv.FormatInt(docs[0].ID, 10), Success: true, Detail: detail,
})
s.logger.Info("upload split at barcode separator pages",
s.reqLog(ctx).Info("upload split at barcode separator pages",
"tenant_id", tenantID, "filename", filename, "pages", res.PageCount,
"separator_pages", res.SeparatorPages, "parts", len(res.Parts),
"documents_created", len(docs), "duplicates_skipped", dupCount)
@@ -1238,7 +1219,7 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
mimeType := detectMimeType("", ext, doc.StoragePath)
result, err := s.ocr.Extract(ctx, doc.StoragePath, mimeType)
if err != nil {
s.logger.Warn("jobqueue ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID,
s.reqLog(ctx).Warn("jobqueue ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID,
"storage_path", doc.StoragePath, "resolved_mime", mimeType, "err", err)
s.audlog.Log(audit.Entry{EventType: audit.EventDocumentProcessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "ocr_failed: " + err.Error()})
return fmt.Errorf("jobqueue ocr extract: %w", err)
@@ -1251,7 +1232,7 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
// retry/backoff) never accumulates duplicates. Best-effort, never fails
// the job since ocr_text is already the authoritative persisted result.
if err := s.store.ReplaceOCRWords(ctx, documentID, ocrWordsFromResult(documentID, result.Words)); err != nil {
s.logger.Warn("jobqueue replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("jobqueue replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
}
if err := s.store.UpdateDocumentOCRText(ctx, documentID, tenantID, ocrText); err != nil {
@@ -1267,7 +1248,7 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
prefix, dateLayout := s.tenantScanTitleParams(ctx, tenantID)
if newTitle := titleFromOCRText(ocrText, prefix, dateLayout); newTitle != "" && newTitle != doc.Title {
if err := s.store.UpdateDocumentTitleAuto(ctx, documentID, tenantID, newTitle); err != nil {
s.logger.Warn("jobqueue title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("jobqueue title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
} else {
doc.Title = newTitle
}
@@ -1286,7 +1267,7 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
}
if !sameDate(datePtr, doc.DocumentDate) {
if err := s.store.UpdateDocumentDate(ctx, documentID, tenantID, datePtr, scorePtr); err != nil {
s.logger.Warn("jobqueue document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("jobqueue document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
} else {
doc.DocumentDate = datePtr
doc.DocumentDateScore = scorePtr
@@ -1295,11 +1276,11 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
}
if assignWarn := s.autoAssignTaxonomy(ctx, tenantID, doc, barcodes, ocrText); assignWarn != "" {
s.logger.Info("jobqueue auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn)
s.reqLog(ctx).Info("jobqueue auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn)
}
if err := s.store.RunWorkflowsForDocument(ctx, tenantID, doc, storage.WorkflowTriggerOnUpload); err != nil {
s.logger.Warn("jobqueue workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("jobqueue workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
}
// Final index sync: auto-assignment/workflows may have re-indexed already,
@@ -1323,7 +1304,7 @@ func (s *Server) autoAssignTaxonomy(ctx context.Context, tenantID int64, doc *st
// Nachvollziehbarkeit even when nothing matched.
if len(barcodes) > 0 {
if err := s.store.SetDocumentBarcodeValues(ctx, doc.ID, tenantID, barcodes); err != nil {
s.logger.Warn("failed to store barcode values", "document_id", doc.ID, "err", err)
s.reqLog(ctx).Warn("failed to store barcode values", "document_id", doc.ID, "err", err)
}
}
@@ -1371,7 +1352,7 @@ func (s *Server) autoAssignTaxonomy(ctx context.Context, tenantID int64, doc *st
for _, kind := range taxonomyKinds {
entities, err := s.store.ListActiveMatchers(ctx, kind, tenantID)
if err != nil {
s.logger.Warn("failed to list active matchers", "kind", kind, "err", err)
s.reqLog(ctx).Warn("failed to list active matchers", "kind", kind, "err", err)
continue
}
for _, entity := range entities {
@@ -1537,31 +1518,9 @@ func randomUploadID() string {
return hex.EncodeToString(b)
}
// copyFile is the cross-device fallback for os.Rename (EXDEV): copy + fsync
// + remove the source.
func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o640)
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
out.Close()
os.Remove(dst)
return err
}
if err := out.Sync(); err != nil {
out.Close()
os.Remove(dst)
return err
}
return out.Close()
}
// The cross-device copy fallback for the WORM move lives in
// internal/objectstore (copyFile there) since FDN-03 moved the archival step
// behind the storage abstraction.
// ocrSupportedMimeTypes is the whitelist of MIME types the OCR pipeline
// (internal/ocr.Extract) actually dispatches on. A declared Content-Type is
+155
View File
@@ -0,0 +1,155 @@
// FDN-08 — /metrics im Prometheus-Textformat, ohne Fremdabhängigkeit.
//
// Zugriff: bewusst OHNE Login (Scrape-Clients haben keine Session), dafür
// IP-beschränkt. Default ist loopback-only; weitere Scraper werden über
// config api.metrics_allowed_ips (IP oder CIDR) freigeschaltet. Es werden
// ausschließlich aggregierte Zähler ausgegeben — keine Tenant-Daten, keine
// Pfadsegmente mit IDs oder Tokens (siehe normalizeRoute).
package api
import (
"context"
"fmt"
"net"
"net/http"
"runtime"
"sort"
"strings"
"time"
)
// handleMetrics rendert die Registry im Prometheus-Textformat.
func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) {
if !s.metricsAllowed(r) {
writeError(w, http.StatusForbidden, "metrics endpoint not allowed from this address")
return
}
routes, inFlight, panics := s.metrics.snapshot()
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
w.WriteHeader(http.StatusOK)
var b strings.Builder
fmt.Fprintf(&b, "# HELP archivdms_build_info Statische Build-Information.\n")
fmt.Fprintf(&b, "# TYPE archivdms_build_info gauge\n")
fmt.Fprintf(&b, "archivdms_build_info{version=\"%s\"} 1\n", escapeLabel(s.appVersion))
fmt.Fprintf(&b, "# HELP archivdms_uptime_seconds Laufzeit des Prozesses in Sekunden.\n")
fmt.Fprintf(&b, "# TYPE archivdms_uptime_seconds gauge\n")
fmt.Fprintf(&b, "archivdms_uptime_seconds %.3f\n", time.Since(s.startTime).Seconds())
fmt.Fprintf(&b, "# HELP archivdms_goroutines Aktuelle Anzahl Goroutinen.\n")
fmt.Fprintf(&b, "# TYPE archivdms_goroutines gauge\n")
fmt.Fprintf(&b, "archivdms_goroutines %d\n", runtime.NumGoroutine())
fmt.Fprintf(&b, "# HELP archivdms_http_requests_in_flight Aktuell laufende HTTP-Anfragen.\n")
fmt.Fprintf(&b, "# TYPE archivdms_http_requests_in_flight gauge\n")
fmt.Fprintf(&b, "archivdms_http_requests_in_flight %d\n", inFlight)
fmt.Fprintf(&b, "# HELP archivdms_panics_total Zentral abgefangene Panics (unbehandelte Fehler).\n")
fmt.Fprintf(&b, "# TYPE archivdms_panics_total counter\n")
fmt.Fprintf(&b, "archivdms_panics_total %d\n", panics)
// Requests + Latenz je Route/Status.
keys := make([]routeKey, 0, len(routes))
for k := range routes {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
if keys[i].route != keys[j].route {
return keys[i].route < keys[j].route
}
if keys[i].method != keys[j].method {
return keys[i].method < keys[j].method
}
return keys[i].status < keys[j].status
})
fmt.Fprintf(&b, "# HELP archivdms_http_requests_total Anzahl HTTP-Anfragen je Route und Status.\n")
fmt.Fprintf(&b, "# TYPE archivdms_http_requests_total counter\n")
for _, k := range keys {
st := routes[k]
fmt.Fprintf(&b, "archivdms_http_requests_total{method=\"%s\",route=\"%s\",status=\"%d\"} %d\n",
escapeLabel(k.method), escapeLabel(k.route), k.status, st.count)
}
fmt.Fprintf(&b, "# HELP archivdms_http_request_duration_seconds Latenz der HTTP-Anfragen.\n")
fmt.Fprintf(&b, "# TYPE archivdms_http_request_duration_seconds histogram\n")
for _, k := range keys {
st := routes[k]
labels := fmt.Sprintf("method=\"%s\",route=\"%s\",status=\"%d\"",
escapeLabel(k.method), escapeLabel(k.route), k.status)
for i, ub := range latencyBuckets {
fmt.Fprintf(&b, "archivdms_http_request_duration_seconds_bucket{%s,le=\"%g\"} %d\n",
labels, ub, st.bucketCount[i])
}
fmt.Fprintf(&b, "archivdms_http_request_duration_seconds_bucket{%s,le=\"+Inf\"} %d\n", labels, st.count)
fmt.Fprintf(&b, "archivdms_http_request_duration_seconds_sum{%s} %.6f\n", labels, st.sumSeconds)
fmt.Fprintf(&b, "archivdms_http_request_duration_seconds_count{%s} %d\n", labels, st.count)
}
// Queue-Länge (Akzeptanzkriterium 2). Fehler hier dürfen den Scrape nicht
// scheitern lassen — dann fehlt die Metrik einfach für diesen Durchlauf.
if s.store != nil {
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
defer cancel()
counts, err := s.store.CountProcessingJobsByStatus(ctx)
if err != nil {
s.reqLog(r.Context()).Warn("metrics: queue length query failed", "err", err)
} else {
fmt.Fprintf(&b, "# HELP archivdms_processing_jobs Länge der Verarbeitungswarteschlange je Status.\n")
fmt.Fprintf(&b, "# TYPE archivdms_processing_jobs gauge\n")
statuses := make([]string, 0, len(counts))
for st := range counts {
statuses = append(statuses, st)
}
sort.Strings(statuses)
for _, st := range statuses {
fmt.Fprintf(&b, "archivdms_processing_jobs{status=\"%s\"} %d\n", escapeLabel(st), counts[st])
}
}
}
w.Write([]byte(b.String()))
}
// metricsAllowed prüft die Herkunft des Scrape-Requests: loopback immer,
// sonst nur konfigurierte IPs/CIDRs (config api.metrics_allowed_ips).
func (s *Server) metricsAllowed(r *http.Request) bool {
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
host = r.RemoteAddr
}
ip := net.ParseIP(strings.TrimSpace(host))
if ip == nil {
return false
}
if ip.IsLoopback() {
return true
}
for _, entry := range s.cfg.MetricsAllowedIPs {
entry = strings.TrimSpace(entry)
if entry == "" {
continue
}
if strings.Contains(entry, "/") {
if _, cidr, err := net.ParseCIDR(entry); err == nil && cidr.Contains(ip) {
return true
}
continue
}
if parsed := net.ParseIP(entry); parsed != nil && parsed.Equal(ip) {
return true
}
}
return false
}
// escapeLabel entschärft Anführungszeichen/Backslashes/Zeilenumbrüche in
// Prometheus-Labelwerten.
func escapeLabel(v string) string {
r := strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`)
return r.Replace(v)
}
+404
View File
@@ -0,0 +1,404 @@
// FDN-08 — Logging, Metriken & Fehler-Tracking.
//
// Dieses File enthält die drei Bausteine, die bisher gefehlt haben:
//
// 1. requestIDMiddleware: erzeugt (oder übernimmt aus X-Request-ID) eine
// Korrelations-ID, hängt sie an den context.Context und gibt sie im
// Response-Header zurück. Über loggerFromCtx(ctx) bekommt jede Log-Zeile
// im Request-Lebenszyklus das Feld request_id, ohne dass jede Call-Site
// umgeschrieben werden muss.
// 2. metricsMiddleware: zählt Requests je (Methode, normalisierter Pfad,
// Status) und summiert die Latenz in Histogramm-Buckets. Ausgabe über
// GET /metrics im Prometheus-Textformat (kein prometheus/client_golang).
// 3. recoverMiddleware: zentrales Panic-Recovery. net/http's ServeMux hat
// keins; ohne das reißt ein Panic in einem Handler die Verbindung ab und
// der Fehler taucht nirgends auf.
//
// Bewusst KEIN Logging von Query-Strings, Headern oder Request-Bodies:
// dort stecken Tokens (Signed-URL-Signatur, Share-Token, Bearer-Keys) und
// potenziell Passwörter. Geloggt werden ausschließlich Methode, normalisierter
// Pfad (IDs/Tokens durch Platzhalter ersetzt), Status, Dauer und Client-IP.
package api
import (
"context"
"crypto/rand"
"encoding/hex"
"log/slog"
"net/http"
"runtime/debug"
"strconv"
"strings"
"sync"
"time"
)
// stackTrace liefert einen gekürzten Stacktrace für das Panic-Log.
func stackTrace() string {
const maxStack = 4096
buf := debug.Stack()
if len(buf) > maxStack {
buf = buf[:maxStack]
}
return string(buf)
}
const (
requestIDKey contextKey = "request_id"
loggerKey contextKey = "logger"
requestIDHeader = "X-Request-ID"
maxRequestIDLen = 64
)
// newRequestID erzeugt eine zufällige 16-stellige Hex-ID. Fällt bei einem
// (praktisch unmöglichen) Fehler der Entropiequelle auf einen Zeitstempel
// zurück — eine Anfrage darf daran nie scheitern.
func newRequestID() string {
buf := make([]byte, 8)
if _, err := rand.Read(buf); err != nil {
return strconv.FormatInt(time.Now().UnixNano(), 36)
}
return hex.EncodeToString(buf)
}
// sanitizeRequestID übernimmt eine vom Client/Proxy gelieferte ID nur, wenn
// sie kurz und druckbar-alphanumerisch ist. Verhindert Log-Injection über
// Zeilenumbrüche im Header.
func sanitizeRequestID(v string) string {
v = strings.TrimSpace(v)
if v == "" || len(v) > maxRequestIDLen {
return ""
}
for _, r := range v {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
case r == '-' || r == '_' || r == '.':
default:
return ""
}
}
return v
}
// requestIDFromCtx liefert die Korrelations-ID der laufenden Anfrage oder "".
func requestIDFromCtx(ctx context.Context) string {
if ctx == nil {
return ""
}
v, _ := ctx.Value(requestIDKey).(string)
return v
}
// loggerFromCtx liefert den Request-Logger (inkl. request_id-Feld). Außerhalb
// eines HTTP-Requests — oder wenn kein Logger hinterlegt wurde — kommt ein
// no-op-freier Fallback zurück, damit Aufrufer nie auf nil prüfen müssen.
func loggerFromCtx(ctx context.Context) *slog.Logger {
if ctx != nil {
if l, ok := ctx.Value(loggerKey).(*slog.Logger); ok && l != nil {
return l
}
}
return slog.Default()
}
// reqLog ist der Einstieg für Handler: nutzt den Request-Logger aus dem
// Context, fällt aber auf den Server-Logger zurück, wenn die Middleware nicht
// durchlaufen wurde (z.B. in Tests, die Handler direkt aufrufen).
func (s *Server) reqLog(ctx context.Context) *slog.Logger {
if ctx != nil {
if l, ok := ctx.Value(loggerKey).(*slog.Logger); ok && l != nil {
return l
}
}
if s.logger != nil {
return s.logger
}
return slog.Default()
}
// requestIDMiddleware setzt die Korrelations-ID und den Request-Logger.
func (s *Server) requestIDMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rid := sanitizeRequestID(r.Header.Get(requestIDHeader))
if rid == "" {
rid = newRequestID()
}
base := s.logger
if base == nil {
base = slog.Default()
}
reqLogger := base.With("request_id", rid)
ctx := context.WithValue(r.Context(), requestIDKey, rid)
ctx = context.WithValue(ctx, loggerKey, reqLogger)
w.Header().Set(requestIDHeader, rid)
next.ServeHTTP(w, r.WithContext(ctx))
})
}
// statusRecorder merkt sich Statuscode und geschriebene Bytes, damit die
// Metrik-Middleware nach dem Handler auswerten kann.
type statusRecorder struct {
http.ResponseWriter
status int
written int64
wrote bool
}
func (rec *statusRecorder) WriteHeader(code int) {
if !rec.wrote {
rec.status = code
rec.wrote = true
}
rec.ResponseWriter.WriteHeader(code)
}
func (rec *statusRecorder) Write(b []byte) (int, error) {
if !rec.wrote {
rec.status = http.StatusOK
rec.wrote = true
}
n, err := rec.ResponseWriter.Write(b)
rec.written += int64(n)
return n, err
}
// Flush reicht http.Flusher durch (Downloads/Streaming-Endpunkte).
func (rec *statusRecorder) Flush() {
if f, ok := rec.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
// recoverMiddleware fängt Panics zentral ab: Log mit Korrelations-ID +
// Stacktrace, sauberer 500 an den Client. Ohne das stürzt zwar nicht der
// Prozess (net/http fängt pro Verbindung ab), der Fehler bleibt aber
// unsichtbar und der Client bekommt einen abgebrochenen Stream.
func (s *Server) recoverMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
rec := recover()
if rec == nil {
return
}
if rec == http.ErrAbortHandler {
panic(rec)
}
s.metrics.incPanic()
loggerFromCtx(r.Context()).Error("panic in http handler",
"method", r.Method,
"path", normalizeRoute(r.URL.Path),
"remote_ip", s.remoteIP(r),
"panic", rec,
"stack", stackTrace(),
)
if sr, ok := w.(*statusRecorder); ok && sr.wrote {
return // Header sind raus, mehr geht nicht
}
writeError(w, http.StatusInternalServerError, "interner Serverfehler")
}()
next.ServeHTTP(w, r)
})
}
// metricsMiddleware misst Dauer und Status jeder Anfrage.
func (s *Server) metricsMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
start := time.Now()
s.metrics.incInFlight()
defer func() {
s.metrics.decInFlight()
d := time.Since(start)
route := normalizeRoute(r.URL.Path)
s.metrics.observe(r.Method, route, rec.status, d)
// Zugriffs-Log: JEDE Anfrage erzeugt genau eine Zeile mit
// request_id (AK1). Level nach Status gestaffelt, damit im
// Normalbetrieb (Info) nur Auffälligkeiten sichtbar sind:
// 5xx=Error, 4xx=Warn, Rest=Debug.
lvl := slog.LevelDebug
msg := "request completed"
switch {
case rec.status >= 500:
lvl, msg = slog.LevelError, "request failed"
case rec.status >= 400:
lvl, msg = slog.LevelWarn, "request rejected"
}
loggerFromCtx(r.Context()).Log(r.Context(), lvl, msg,
"method", r.Method, "route", route,
"status", rec.status, "duration_ms", d.Milliseconds(),
"bytes", rec.written,
"remote_ip", s.remoteIP(r))
}()
next.ServeHTTP(rec, r)
})
}
// --- Pfad-Normalisierung ---
// tokenSegments sind Pfadabschnitte, deren FOLGENDES Segment ein Geheimnis
// ist (Share-Token). Die dürfen niemals in Logs oder Metrik-Labels landen.
var tokenSegments = map[string]bool{"share": true}
// normalizeRoute ersetzt variable Pfadsegmente durch Platzhalter. Das hält
// die Label-Kardinalität der Metriken klein UND verhindert, dass IDs oder
// Share-Tokens in Logs/Metriken auftauchen. Query-Strings werden nie
// betrachtet (dort stehen Signaturen und API-Keys).
func normalizeRoute(path string) string {
if path == "" {
return "/"
}
parts := strings.Split(path, "/")
prevSecret := false
for i, p := range parts {
if p == "" {
continue
}
if prevSecret {
parts[i] = "{token}"
prevSecret = false
continue
}
prevSecret = tokenSegments[p]
if isVariableSegment(p) {
parts[i] = "{id}"
}
}
out := strings.Join(parts, "/")
if len(out) > 120 {
return "/other"
}
return out
}
// isVariableSegment erkennt IDs, Hashes und sonstige nicht-statische
// Pfadbestandteile. Konservativ: im Zweifel maskieren.
func isVariableSegment(seg string) bool {
if _, err := strconv.ParseInt(seg, 10, 64); err == nil {
return true
}
if len(seg) > 40 {
return true
}
hasDigit := false
for _, r := range seg {
switch {
case r >= '0' && r <= '9':
hasDigit = true
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r == '-', r == '_', r == '.':
default:
// Alles Ungewöhnliche (Sonderzeichen, Umlaute, %-Encoding) ist mit
// Sicherheit kein statisches Routensegment.
return true
}
}
// Statische Routensegmente sind reine Wörter ("delete-requests",
// "classification-templates"). Längere Mischungen aus Buchstaben und
// Ziffern sind Hashes/Tokens/Dateinamen -> maskieren.
return hasDigit && len(seg) >= 8
}
// --- Metrik-Registry ---
// latencyBuckets sind die oberen Grenzen (Sekunden) des Latenz-Histogramms.
var latencyBuckets = []float64{0.005, 0.025, 0.1, 0.5, 1, 2.5, 5, 10, 30}
type routeKey struct {
method string
route string
status int
}
type routeStat struct {
count uint64
sumSeconds float64
bucketCount []uint64 // len(latencyBuckets), kumulativ erst beim Rendern
}
// metricsRegistry ist eine minimale, prozesslokale Metrik-Sammlung. Keine
// globale Variable: hängt als Feld am Server (Dependency Injection).
type metricsRegistry struct {
mu sync.Mutex
routes map[routeKey]*routeStat
inFlight int64
panics uint64
started time.Time
}
func newMetricsRegistry() *metricsRegistry {
return &metricsRegistry{
routes: make(map[routeKey]*routeStat),
started: time.Now(),
}
}
func (m *metricsRegistry) observe(method, route string, status int, d time.Duration) {
if m == nil {
return
}
k := routeKey{method: method, route: route, status: status}
secs := d.Seconds()
m.mu.Lock()
defer m.mu.Unlock()
// Kardinalitätsbremse: unbekannte Pfade nicht unbegrenzt sammeln.
st := m.routes[k]
if st == nil {
if len(m.routes) >= 500 {
k = routeKey{method: method, route: "/other", status: status}
st = m.routes[k]
}
if st == nil {
st = &routeStat{bucketCount: make([]uint64, len(latencyBuckets))}
m.routes[k] = st
}
}
st.count++
st.sumSeconds += secs
for i, ub := range latencyBuckets {
if secs <= ub {
st.bucketCount[i]++
}
}
}
func (m *metricsRegistry) incInFlight() {
if m == nil {
return
}
m.mu.Lock()
m.inFlight++
m.mu.Unlock()
}
func (m *metricsRegistry) decInFlight() {
if m == nil {
return
}
m.mu.Lock()
m.inFlight--
m.mu.Unlock()
}
func (m *metricsRegistry) incPanic() {
if m == nil {
return
}
m.mu.Lock()
m.panics++
m.mu.Unlock()
}
// snapshot liefert eine Kopie der Zähler für das Rendern.
func (m *metricsRegistry) snapshot() (map[routeKey]routeStat, int64, uint64) {
m.mu.Lock()
defer m.mu.Unlock()
out := make(map[routeKey]routeStat, len(m.routes))
for k, v := range m.routes {
cp := routeStat{count: v.count, sumSeconds: v.sumSeconds, bucketCount: make([]uint64, len(v.bucketCount))}
copy(cp.bucketCount, v.bucketCount)
out[k] = cp
}
return out, m.inFlight, m.panics
}
+174
View File
@@ -0,0 +1,174 @@
// FDN-08 — Tests zu den drei Akzeptanzkriterien:
// 1. Korrelations-ID über alle Schichten
// 2. Metriken (Latenz, Fehlerrate, Queue-Länge)
// 3. Unbehandelte Fehler werden zentral gemeldet
//
// Zusätzlich Prüfung 2 der Abnahme: es dürfen keine Tokens/Passwörter in Logs
// oder Metrik-Labels landen (TestNormalizeRouteRedactsSecrets).
package api
import (
"bytes"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"testing"
"archivdms/config"
)
func newTestServer(buf *bytes.Buffer) *Server {
logger := slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
return New(config.APIConfig{}, nil, nil, nil, nil, logger)
}
// AK 1: jede Anfrage bekommt eine Korrelations-ID, eine vom Client gelieferte
// wird übernommen und im Response-Header zurückgegeben.
func TestRequestIDGeneratedAndEchoed(t *testing.T) {
srv := newTestServer(&bytes.Buffer{})
rec := httptest.NewRecorder()
srv.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/health", nil))
got := rec.Header().Get(requestIDHeader)
if got == "" {
t.Fatalf("erwartete generierte Request-ID im Header %s", requestIDHeader)
}
req := httptest.NewRequest(http.MethodGet, "/api/health", nil)
req.Header.Set(requestIDHeader, "abc-123")
rec2 := httptest.NewRecorder()
srv.ServeHTTP(rec2, req)
if rec2.Header().Get(requestIDHeader) != "abc-123" {
t.Fatalf("Client-Request-ID nicht übernommen: %q", rec2.Header().Get(requestIDHeader))
}
// Log-Injection: unsaubere IDs werden verworfen, nicht durchgereicht.
bad := httptest.NewRequest(http.MethodGet, "/api/health", nil)
bad.Header.Set(requestIDHeader, "evil\nid")
rec3 := httptest.NewRecorder()
srv.ServeHTTP(rec3, bad)
if strings.Contains(rec3.Header().Get(requestIDHeader), "evil") {
t.Fatalf("ungültige Request-ID wurde übernommen: %q", rec3.Header().Get(requestIDHeader))
}
}
// AK 1: die ID landet im Logger, den Handler über den Context ziehen.
func TestLoggerFromCtxCarriesRequestID(t *testing.T) {
var buf bytes.Buffer
srv := newTestServer(&buf)
h := srv.requestIDMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if requestIDFromCtx(r.Context()) == "" {
t.Errorf("keine Request-ID im Context")
}
srv.reqLog(r.Context()).Info("testereignis")
}))
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/health", nil))
rid := rec.Header().Get(requestIDHeader)
if !strings.Contains(buf.String(), "request_id="+rid) {
t.Fatalf("Log-Zeile ohne request_id=%s: %s", rid, buf.String())
}
}
// AK 2: Zähler und Latenz-Histogramm werden im Prometheus-Textformat geliefert.
func TestMetricsEndpoint(t *testing.T) {
srv := newTestServer(&bytes.Buffer{})
for i := 0; i < 2; i++ {
srv.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/api/health", nil))
}
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
req.RemoteAddr = "127.0.0.1:54321"
rec := httptest.NewRecorder()
srv.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("erwartet 200, bekam %d", rec.Code)
}
body := rec.Body.String()
want := `archivdms_http_requests_total{method="GET",route="/api/health",status="200"} 2`
if !strings.Contains(body, want) {
t.Fatalf("Request-Zähler fehlt:\n%s", body)
}
for _, frag := range []string{
"archivdms_http_request_duration_seconds_bucket",
"archivdms_http_request_duration_seconds_sum",
"archivdms_goroutines",
"archivdms_panics_total",
} {
if !strings.Contains(body, frag) {
t.Errorf("Metrik %q fehlt in der Ausgabe", frag)
}
}
}
// AK 2 (Zugriffsschutz): fremde IPs dürfen nicht scrapen.
func TestMetricsEndpointIPRestricted(t *testing.T) {
srv := newTestServer(&bytes.Buffer{})
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
req.RemoteAddr = "203.0.113.7:5000"
rec := httptest.NewRecorder()
srv.ServeHTTP(rec, req)
if rec.Code != http.StatusForbidden {
t.Fatalf("erwartet 403 für fremde IP, bekam %d", rec.Code)
}
srv.cfg.MetricsAllowedIPs = []string{"203.0.113.0/24"}
rec2 := httptest.NewRecorder()
req2 := httptest.NewRequest(http.MethodGet, "/metrics", nil)
req2.RemoteAddr = "203.0.113.7:5000"
srv.ServeHTTP(rec2, req2)
if rec2.Code != http.StatusOK {
t.Fatalf("erwartet 200 für freigeschaltete CIDR, bekam %d", rec2.Code)
}
}
// AK 3: ein Panic wird zentral abgefangen, mit Korrelations-ID geloggt und als
// sauberer 500 beantwortet — der Zähler archivdms_panics_total steigt.
func TestRecoverMiddlewareCatchesPanic(t *testing.T) {
var buf bytes.Buffer
srv := newTestServer(&buf)
boom := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
panic("kaputt")
})
h := srv.requestIDMiddleware(srv.metricsMiddleware(srv.recoverMiddleware(boom)))
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/documents/42", nil))
if rec.Code != http.StatusInternalServerError {
t.Fatalf("erwartet 500, bekam %d", rec.Code)
}
rid := rec.Header().Get(requestIDHeader)
logged := buf.String()
if !strings.Contains(logged, "panic in http handler") || !strings.Contains(logged, "request_id="+rid) {
t.Fatalf("Panic nicht mit Korrelations-ID geloggt: %s", logged)
}
if _, _, panics := srv.metrics.snapshot(); panics != 1 {
t.Fatalf("erwartet 1 gezähltes Panic, bekam %d", panics)
}
}
// Abnahme-Prüfung 2: keine Tokens/IDs in Logs oder Metrik-Labels.
func TestNormalizeRouteRedactsSecrets(t *testing.T) {
cases := [][2]string{
{"/api/health", "/api/health"},
{"/api/documents/42", "/api/documents/{id}"},
{"/api/documents/42/notes/7", "/api/documents/{id}/notes/{id}"},
{"/api/classification-templates", "/api/classification-templates"},
{"/api/trash/9/delete-requests", "/api/trash/{id}/delete-requests"},
{"/public/share/s3cr3tTokenXyz", "/public/share/{token}"},
{"/public/share/abc/download", "/public/share/{token}/download"},
}
for _, c := range cases {
in, want := c[0], c[1]
if got := normalizeRoute(in); got != want {
t.Errorf("normalizeRoute(%q) = %q, erwartet %q", in, got, want)
}
}
}
+1 -1
View File
@@ -51,7 +51,7 @@ func (s *Server) handleListDocumentOCRWords(w http.ResponseWriter, r *http.Reque
words, err := s.store.ListOCRWords(r.Context(), id)
if err != nil {
s.logger.Error("list ocr words failed", "document_id", id, "tenant_id", *sess.TenantID, "err", err)
s.reqLog(r.Context()).Error("list ocr words failed", "document_id", id, "tenant_id", *sess.TenantID, "err", err)
writeError(w, http.StatusInternalServerError, "list ocr words failed")
return
}
+5 -6
View File
@@ -15,7 +15,6 @@ import (
"errors"
"io"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
@@ -107,7 +106,7 @@ func (s *Server) handlePublicShareDownload(w http.ResponseWriter, r *http.Reques
// Atomically claim one access slot (closes the max_accesses race).
ok, err := s.store.IncrementShareAccess(r.Context(), rs.ShareID())
if err != nil {
s.logger.Error("share access increment failed", "share_id", rs.ShareID(), "err", err)
s.reqLog(r.Context()).Error("share access increment failed", "share_id", rs.ShareID(), "err", err)
writeError(w, http.StatusInternalServerError, "download failed")
return
}
@@ -118,9 +117,9 @@ func (s *Server) handlePublicShareDownload(w http.ResponseWriter, r *http.Reques
return
}
f, err := os.Open(rs.StoragePath())
f, err := s.objects.Open(r.Context(), rs.TenantID(), rs.StoragePath())
if err != nil {
s.logger.Error("share file open failed", "share_id", rs.ShareID(), "err", err)
s.reqLog(r.Context()).Error("share file open failed", "share_id", rs.ShareID(), "err", err)
writeError(w, http.StatusInternalServerError, "download failed")
return
}
@@ -133,7 +132,7 @@ func (s *Server) handlePublicShareDownload(w http.ResponseWriter, r *http.Reques
w.Header().Set("Content-Disposition", "attachment; filename=\""+safeDownloadName(rs.DocumentTitle, ext)+"\"")
w.Header().Set("X-Content-Type-Options", "nosniff")
if _, err := io.Copy(w, f); err != nil {
s.logger.Warn("share file stream interrupted", "share_id", rs.ShareID(), "err", err)
s.reqLog(r.Context()).Warn("share file stream interrupted", "share_id", rs.ShareID(), "err", err)
}
}
@@ -141,7 +140,7 @@ func (s *Server) handlePublicShareDownload(w http.ResponseWriter, r *http.Reques
// into the audit log (EventShareAccessed). Never blocks the response path.
func (s *Server) recordShareAccess(r *http.Request, rs *storage.ResolvedShare, ip, result string) {
if err := s.store.LogShareAccess(r.Context(), rs.ShareID(), ip, r.UserAgent(), result); err != nil {
s.logger.Error("share access log failed", "share_id", rs.ShareID(), "err", err)
s.reqLog(r.Context()).Error("share access log failed", "share_id", rs.ShareID(), "err", err)
}
tenantID := rs.TenantID()
s.audlog.Log(audit.Entry{
+83 -1
View File
@@ -6,10 +6,13 @@ package api
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"log/slog"
"net"
"net/http"
"strconv"
"strings"
"time"
@@ -19,6 +22,7 @@ import (
"archivdms/internal/ldapauth"
"archivdms/internal/ldapstore"
"archivdms/internal/mailer"
"archivdms/internal/objectstore"
"archivdms/internal/ocr"
"archivdms/internal/pagesplit"
"archivdms/internal/storage"
@@ -46,6 +50,10 @@ type Server struct {
logger *slog.Logger
mux *http.ServeMux
ocr *ocr.Extractor
// objects is the WORM object-storage driver (internal/objectstore): the
// only place archived files are written, read or unlinked. Wired by
// SetStorageConfig/SetObjectStore.
objects objectstore.Store
thumbs *thumbnail.Generator
// pagesplitter performs barcode separator-page splitting of multi-page PDF
// uploads before archival (internal/pagesplit). May be nil / disabled, in
@@ -66,6 +74,15 @@ type Server struct {
// (per client IP) to blunt token/password enumeration.
shareLimiter *ipRateLimiter
// metrics ist die prozesslokale Metrik-Registry (FDN-08,
// internal/api/observability.go). Kein globaler Zustand: hängt am Server.
metrics *metricsRegistry
// baseHandler ist die in New() gebaute Middleware-Kette um s.mux
// (requestID -> metrics -> recover). Einmal gebaut, danach nur gelesen —
// kein Lazy-Init in ServeHTTP (Data Race).
baseHandler http.Handler
// accountingLimiter rate-limits the Bearer-key Buchhaltungs-Pull-API
// (per client IP) to blunt API-key guessing. Separate bucket set from
// shareLimiter so a busy accounting client cannot starve share downloads.
@@ -76,8 +93,46 @@ type Server struct {
// paths, max upload size) into the API server. Needed by
// handleUploadDocument, which cannot rely solely on the storage.Store
// (that only knows its own base dir, not the inbox/ocr-tmp layout).
//
// It also constructs the object-storage driver (internal/objectstore), the
// single place where archived files are written, read and unlinked. Call
// SetFQDN before this if generated signed URLs should be absolute.
func (s *Server) SetStorageConfig(cfg config.StorageConfig) {
s.storageCfg = cfg
secret := s.cfg.Secret
if strings.TrimSpace(secret) == "" {
// No master secret configured: fall back to an ephemeral, per-process
// key so file access keeps working; signed URLs then simply do not
// survive a restart. Never fail startup over this.
buf := make([]byte, 32)
if _, err := rand.Read(buf); err != nil {
secret = strconv.FormatInt(time.Now().UnixNano(), 36)
} else {
secret = hex.EncodeToString(buf)
}
s.logger.Warn("objectstore: api.secret unset, using ephemeral signing key")
}
driver, err := objectstore.NewLocalStore(cfg, secret, s.publicBaseURL())
if err != nil {
s.logger.Error("objectstore init failed", "err", err)
return
}
s.objects = driver
}
// SetObjectStore overrides the object-storage driver (tests / alternative
// wiring). Normally set implicitly by SetStorageConfig.
func (s *Server) SetObjectStore(o objectstore.Store) {
s.objects = o
}
// publicBaseURL is the origin used for generated links. Empty FQDN yields
// site-relative URLs.
func (s *Server) publicBaseURL() string {
if strings.TrimSpace(s.fqdn) == "" {
return ""
}
return "https://" + strings.TrimSuffix(strings.TrimSpace(s.fqdn), "/")
}
// SetOCR wires the OCR extractor into the API server. May be nil, in which
@@ -150,8 +205,10 @@ func New(
shareLimiter: newIPRateLimiter(20, 1.0),
// Batch pulls are legitimate here: 60 requests burst, refilled at 5/sec.
accountingLimiter: newIPRateLimiter(60, 5.0),
metrics: newMetricsRegistry(),
}
s.routes()
s.baseHandler = s.requestIDMiddleware(s.metricsMiddleware(s.recoverMiddleware(s.mux)))
return s
}
@@ -168,6 +225,10 @@ func (s *Server) authAdmin(h http.HandlerFunc) http.HandlerFunc {
func (s *Server) routes() {
s.mux.HandleFunc("GET /api/health", s.handleHealth)
s.mux.HandleFunc("GET /api/version", s.handleVersion)
// Prometheus-Scrape-Endpunkt (FDN-08, internal/api/metrics_handlers.go).
// Bewusst ohne s.auth — ein Scraper hat keine Session; der Zugriff wird
// stattdessen per Quell-IP begrenzt (loopback + api.metrics_allowed_ips).
s.mux.HandleFunc("GET /metrics", s.handleMetrics)
s.mux.HandleFunc("POST /api/auth/login", s.handleLogin)
s.mux.HandleFunc("GET /api/auth/me", s.auth(s.handleMe))
@@ -377,6 +438,12 @@ func (s *Server) routes() {
s.mux.HandleFunc("GET /public/share/{token}", s.handlePublicShareMeta)
s.mux.HandleFunc("POST /public/share/{token}/download", s.handlePublicShareDownload)
// Signed, time-limited download URLs (internal/api/signed_url_handlers.go).
// Issuing is authenticated + tenant-scoped; redeeming runs WITHOUT s.auth
// because the HMAC signature in the query string is the credential.
s.mux.HandleFunc("POST /api/documents/{id}/signed-url", s.auth(s.handleCreateDocumentSignedURL))
s.mux.HandleFunc("GET /public/files", s.handleSignedFileDownload)
// Buchhaltungs-Pull-API (internal/api/accounting_handlers.go).
// Key administration runs on the normal session auth and is domain_admin-only
// (a key grants tenant-wide read access to archived documents).
@@ -417,8 +484,23 @@ func (s *Server) routes() {
}
// ServeHTTP implements http.Handler.
//
// Die Basis-Middleware-Kette (FDN-08) liegt bewusst hier und nicht an den
// einzelnen Routen, damit sie ausnahmslos für JEDE Anfrage gilt — auch für
// /public/*, /metrics und nicht gefundene Pfade:
//
// requestID -> metrics -> recover -> ServeMux
//
// Reihenfolge: requestID zuerst, damit Metrik- und Panic-Log die
// Korrelations-ID haben; recover innen, damit der 500 noch über den
// statusRecorder der Metrik-Middleware läuft und dort gezählt wird.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.mux.ServeHTTP(w, r)
if s.baseHandler == nil {
// Server wurde nicht über New() gebaut (Tests): ohne Kette bedienen.
s.mux.ServeHTTP(w, r)
return
}
s.baseHandler.ServeHTTP(w, r)
}
// --- system handlers ---
+162
View File
@@ -0,0 +1,162 @@
// Signed, time-limited download URLs for archived documents (FDN-03,
// internal/objectstore). Same principle as the external share links
// (share_handlers.go / public_share_handlers.go) — unguessable credential in
// the link, mandatory hard expiry, IP rate limiting, audit trail — but
// stateless: the credential is an HMAC-SHA256 signature over
// tenant|document|expiry instead of a DB row.
//
// POST /api/documents/{id}/signed-url issue a link (authenticated, tenant-scoped)
// GET /public/files?t=&d=&exp=&sig= redeem it (no session, signature is the credential)
//
// Use share links when a document is handed to an external party with its own
// lifecycle (revoke, password, access cap); use signed URLs for short-lived
// machine access to the file itself (viewer, export job) without a session
// cookie. Nothing here bypasses tenant scoping: the tenant id is taken from
// the signed payload and every lookup still filters on it.
package api
import (
"encoding/json"
"errors"
"io"
"net/http"
"path/filepath"
"strconv"
"time"
"archivdms/internal/audit"
"archivdms/internal/objectstore"
)
// createSignedURLRequest is the optional POST body. TTLMinutes overrides the
// configured default validity (storage.signed_url_ttl_minutes); it is capped
// at 24 hours so no effectively unbounded link can be minted.
type createSignedURLRequest struct {
TTLMinutes int `json:"ttl_minutes,omitempty"`
}
// createSignedURLResponse is returned to the caller.
type createSignedURLResponse struct {
URL string `json:"url"`
ExpiresAt time.Time `json:"expires_at"`
}
// maxSignedURLTTL caps a caller-supplied validity.
const maxSignedURLTTL = 24 * time.Hour
// handleCreateDocumentSignedURL handles POST /api/documents/{id}/signed-url.
func (s *Server) handleCreateDocumentSignedURL(w http.ResponseWriter, r *http.Request) {
sess := sessionFromCtx(r.Context())
if sess.TenantID == nil {
writeError(w, http.StatusForbidden, "tenant context required")
return
}
if s.objects == nil {
writeError(w, http.StatusServiceUnavailable, "Dateispeicher nicht konfiguriert")
return
}
docID, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid document id")
return
}
var req createSignedURLRequest
if r.Body != nil {
_ = json.NewDecoder(r.Body).Decode(&req) // body is optional
}
ttl := time.Duration(req.TTLMinutes) * time.Minute
if ttl <= 0 {
ttl = s.storageCfg.ResolvedSignedURLTTL()
}
if ttl > maxSignedURLTTL {
ttl = maxSignedURLTTL
}
// IDOR guard: the document must belong to the caller's tenant. GetDocument
// filters WHERE id = $1 AND tenant_id = $2.
doc, err := s.store.GetDocument(r.Context(), docID, *sess.TenantID)
if err != nil {
s.logShare(r, audit.EventSignedURLCreated, sess.TenantID, sess.Username,
"signed_url doc:"+strconv.FormatInt(docID, 10)+" err:not_found", false)
writeError(w, http.StatusNotFound, "document not found")
return
}
link, err := s.objects.SignedURL(*sess.TenantID, doc.ID, ttl)
if err != nil {
s.logShare(r, audit.EventSignedURLCreated, sess.TenantID, sess.Username,
"signed_url doc:"+strconv.FormatInt(docID, 10)+" err:"+err.Error(), false)
writeError(w, http.StatusInternalServerError, "Link konnte nicht erstellt werden")
return
}
expiresAt := time.Now().Add(ttl)
s.logShare(r, audit.EventSignedURLCreated, sess.TenantID, sess.Username,
"signed_url doc:"+strconv.FormatInt(docID, 10)+" ttl:"+ttl.String(), true)
writeJSON(w, http.StatusOK, createSignedURLResponse{URL: link, ExpiresAt: expiresAt})
}
// handleSignedFileDownload handles GET /public/files?t=&d=&exp=&sig=. Served
// WITHOUT the s.auth wrapper by design: the signature is the credential.
// Order: rate-limit -> verify signature -> verify expiry -> tenant-scoped
// document lookup -> stream from the WORM store. Every outcome is audit-logged
// (EventSignedURLAccessed), successes and failures alike.
func (s *Server) handleSignedFileDownload(w http.ResponseWriter, r *http.Request) {
ip := s.remoteIP(r)
if !s.shareLimiter.allow(ip) {
writeError(w, http.StatusTooManyRequests, "too many requests")
return
}
if s.objects == nil {
writeError(w, http.StatusServiceUnavailable, "Dateispeicher nicht konfiguriert")
return
}
ref, err := s.objects.VerifySignedURL(r.URL.Query(), time.Now())
if err != nil {
s.logSignedAccess(r, nil, "signed_url_access err:"+err.Error(), false)
if errors.Is(err, objectstore.ErrSignatureExpired) {
writeError(w, http.StatusGone, "Der Link ist abgelaufen.")
return
}
writeError(w, http.StatusForbidden, "Der Link ist ungültig.")
return
}
doc, err := s.store.GetDocument(r.Context(), ref.DocumentID, ref.TenantID)
if err != nil {
s.logSignedAccess(r, &ref.TenantID,
"signed_url_access doc:"+strconv.FormatInt(ref.DocumentID, 10)+" err:not_found", false)
writeError(w, http.StatusNotFound, "Dokument nicht gefunden.")
return
}
f, err := s.objects.Open(r.Context(), ref.TenantID, doc.StoragePath)
if err != nil {
s.reqLog(r.Context()).Error("signed url file open failed", "document_id", doc.ID, "tenant_id", ref.TenantID, "err", err)
s.logSignedAccess(r, &ref.TenantID,
"signed_url_access doc:"+strconv.FormatInt(doc.ID, 10)+" err:"+err.Error(), false)
if errors.Is(err, objectstore.ErrObjectNotFound) {
writeError(w, http.StatusNotFound, "Datei nicht im Archiv vorhanden.")
return
}
writeError(w, http.StatusInternalServerError, "Download fehlgeschlagen.")
return
}
defer f.Close()
s.logSignedAccess(r, &ref.TenantID, "signed_url_access doc:"+strconv.FormatInt(doc.ID, 10), true)
ext := filepath.Ext(doc.StoragePath)
w.Header().Set("Content-Type", detectMimeType("", ext, doc.StoragePath))
w.Header().Set("Content-Disposition", "attachment; filename=\""+safeDownloadName(doc.Title, ext)+"\"")
w.Header().Set("X-Content-Type-Options", "nosniff")
if _, err := io.Copy(w, f); err != nil {
s.reqLog(r.Context()).Warn("signed url stream interrupted", "document_id", doc.ID, "err", err)
}
}
// logSignedAccess records a redemption attempt in the audit log. There is no
// session here, so the username column carries the anonymous marker.
func (s *Server) logSignedAccess(r *http.Request, tenantID *int64, detail string, ok bool) {
s.audlog.Log(audit.Entry{
EventType: audit.EventSignedURLAccessed, Username: "anonymous", TenantID: tenantID,
IPAddress: s.remoteIP(r), Success: ok, Detail: detail,
})
}
+6
View File
@@ -87,6 +87,12 @@ const (
EventShareRevoked = "share_revoked"
EventShareAccessed = "share_accessed"
// Signed, time-limited storage download URLs (internal/objectstore,
// internal/api/signed_url_handlers.go). Created is logged when a link is
// issued, Accessed on every attempt to redeem one (success and failure).
EventSignedURLCreated = "signed_url_created"
EventSignedURLAccessed = "signed_url_accessed"
// LDAP directory integration (internal/ldapstore, internal/ldapauth,
// internal/api/ldap_handlers.go). ConfigChanged covers create/update/delete
// and the test action; LoginSuccess/Failed are logged on every LDAP bind
+247
View File
@@ -0,0 +1,247 @@
package objectstore
import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"golang.org/x/crypto/hkdf"
"archivdms/config"
)
// signKeyInfo domain-separates the download-URL signing key from every other
// key derived from the same master secret (JWT uses "archivdms-jwt-v1", the
// LDAP secretbox uses "archivdms-ldap-secretbox-v1").
const signKeyInfo = "archivdms-storage-url-v1"
// SignedPath is the route a signed download URL points at. It is served
// WITHOUT session auth — the signature is the credential.
const SignedPath = "/public/files"
// LocalStore is the local-filesystem WORM driver and the only implementation
// of Store. It owns no state beyond the storage configuration, the URL signing
// key and the public base URL used for link generation.
type LocalStore struct {
cfg config.StorageConfig
signKey []byte
baseURL string
}
// compile-time interface check.
var _ Store = (*LocalStore)(nil)
// NewLocalStore builds the local driver. secret is the application master
// secret (config.APIConfig.Secret) from which the URL signing key is derived
// via HKDF-SHA256; baseURL is the public origin used for generated links
// (empty = emit site-relative URLs).
func NewLocalStore(cfg config.StorageConfig, secret, baseURL string) (*LocalStore, error) {
if strings.TrimSpace(secret) == "" {
return nil, fmt.Errorf("objectstore: empty signing secret")
}
key := make([]byte, 32)
if _, err := io.ReadFull(hkdf.New(sha256.New, []byte(secret), nil, []byte(signKeyInfo)), key); err != nil {
return nil, fmt.Errorf("objectstore: derive signing key: %w", err)
}
return &LocalStore{
cfg: cfg,
signKey: key,
baseURL: strings.TrimRight(strings.TrimSpace(baseURL), "/"),
}, nil
}
// tenantRoot is the tenant's WORM subtree: <BasePath>/store/<tenant_id>.
func (l *LocalStore) tenantRoot(tenantID int64) string {
return filepath.Join(l.cfg.StorePath(), strconv.FormatInt(tenantID, 10))
}
// resolveTenantPath cleans storagePath and verifies it lies inside the
// tenant's own store subtree. This is the filesystem-level counterpart of the
// "WHERE tenant_id = $N" rule: even a manipulated storage_path from the DB
// cannot be used to read another tenant's archive.
func (l *LocalStore) resolveTenantPath(tenantID int64, storagePath string) (string, error) {
if strings.TrimSpace(storagePath) == "" {
return "", ErrObjectNotFound
}
clean := filepath.Clean(storagePath)
root := filepath.Clean(l.tenantRoot(tenantID))
if clean != root && !strings.HasPrefix(clean, root+string(os.PathSeparator)) {
return "", ErrOutsideTenant
}
return clean, nil
}
// Archive implements Store. It reproduces, unchanged, the archival steps the
// upload pipeline has always performed: build store/<tenant>/<yyyy>/<mm>,
// reject an existing target as duplicate, move (rename, copy+remove fallback
// across devices) and finally chmod 0440 — the one and only chmod, applied
// once the file sits at its final path.
func (l *LocalStore) Archive(ctx context.Context, tenantID int64, srcPath, ext, contentHash string, at time.Time) (string, error) {
if at.IsZero() {
at = time.Now()
}
storeDir := filepath.Join(l.tenantRoot(tenantID),
fmt.Sprintf("%04d", at.Year()), fmt.Sprintf("%02d", at.Month()))
if err := os.MkdirAll(storeDir, 0o750); err != nil {
os.Remove(srcPath)
return "", fmt.Errorf("objectstore: create store dir: %w", err)
}
dst := filepath.Join(storeDir, contentHash+ext)
// Collision check: identical hash already stored -> duplicate.
if _, err := os.Stat(dst); err == nil {
os.Remove(srcPath)
return "", ErrObjectExists
} else if !os.IsNotExist(err) {
os.Remove(srcPath)
return "", fmt.Errorf("objectstore: stat store path: %w", err)
}
if err := os.Rename(srcPath, dst); err != nil {
if copyErr := copyFile(srcPath, dst); copyErr != nil {
os.Remove(srcPath)
return "", fmt.Errorf("objectstore: move file to store: rename failed (%v), copy fallback failed: %w", err, copyErr)
}
os.Remove(srcPath)
}
// WORM lock: read-only for everyone from now on.
if err := os.Chmod(dst, 0o440); err != nil {
return "", fmt.Errorf("objectstore: chmod store file: %w", err)
}
return dst, nil
}
// Open implements Store.
func (l *LocalStore) Open(ctx context.Context, tenantID int64, storagePath string) (io.ReadSeekCloser, error) {
p, err := l.resolveTenantPath(tenantID, storagePath)
if err != nil {
return nil, err
}
f, err := os.Open(p)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("objectstore: open %q: %w", p, ErrObjectNotFound)
}
return nil, fmt.Errorf("objectstore: open object: %w", err)
}
return f, nil
}
// Stat implements Store.
func (l *LocalStore) Stat(ctx context.Context, tenantID int64, storagePath string) (os.FileInfo, error) {
p, err := l.resolveTenantPath(tenantID, storagePath)
if err != nil {
return nil, err
}
fi, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("objectstore: stat %q: %w", p, ErrObjectNotFound)
}
return nil, fmt.Errorf("objectstore: stat object: %w", err)
}
return fi, nil
}
// Delete implements Store. Only legitimate after a confirmed deletion request
// whose retention period has expired — this layer performs no retention check
// of its own; that stays in storage.ConfirmDeleteRequest, which still unlinks
// inside its own transaction and is deliberately left untouched by FDN-03
// (moving it here would mean handing the DB layer a filesystem driver).
func (l *LocalStore) Delete(ctx context.Context, tenantID int64, storagePath string) error {
p, err := l.resolveTenantPath(tenantID, storagePath)
if err != nil {
return err
}
if err := os.Remove(p); err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("objectstore: remove %q: %w", p, ErrObjectNotFound)
}
return fmt.Errorf("objectstore: remove object: %w", err)
}
return nil
}
// SignedURL implements Store. The URL carries tenant id, document id and an
// absolute expiry, authenticated by an HMAC-SHA256 over exactly those three
// values — the same "unguessable token, hard expiry, server-side check"
// principle as the external share links, only stateless (no DB row).
func (l *LocalStore) SignedURL(tenantID, documentID int64, ttl time.Duration) (string, error) {
if tenantID <= 0 || documentID <= 0 {
return "", fmt.Errorf("objectstore: invalid signed url reference")
}
if ttl <= 0 {
ttl = l.cfg.ResolvedSignedURLTTL()
}
exp := time.Now().Add(ttl).Unix()
q := url.Values{}
q.Set("t", strconv.FormatInt(tenantID, 10))
q.Set("d", strconv.FormatInt(documentID, 10))
q.Set("exp", strconv.FormatInt(exp, 10))
q.Set("sig", l.sign(tenantID, documentID, exp))
return l.baseURL + SignedPath + "?" + q.Encode(), nil
}
// VerifySignedURL implements Store. Signature first, expiry second, so a
// forged link never learns anything from the expiry branch.
func (l *LocalStore) VerifySignedURL(q url.Values, now time.Time) (SignedRef, error) {
tenantID, err1 := strconv.ParseInt(q.Get("t"), 10, 64)
documentID, err2 := strconv.ParseInt(q.Get("d"), 10, 64)
exp, err3 := strconv.ParseInt(q.Get("exp"), 10, 64)
sig := q.Get("sig")
if err1 != nil || err2 != nil || err3 != nil || sig == "" || tenantID <= 0 || documentID <= 0 {
return SignedRef{}, ErrSignatureInvalid
}
want := l.sign(tenantID, documentID, exp)
if !hmac.Equal([]byte(want), []byte(sig)) {
return SignedRef{}, ErrSignatureInvalid
}
expiresAt := time.Unix(exp, 0)
if !now.Before(expiresAt) {
return SignedRef{}, ErrSignatureExpired
}
return SignedRef{TenantID: tenantID, DocumentID: documentID, ExpiresAt: expiresAt}, nil
}
// sign returns the base64url HMAC-SHA256 over the canonical payload.
func (l *LocalStore) sign(tenantID, documentID, exp int64) string {
mac := hmac.New(sha256.New, l.signKey)
fmt.Fprintf(mac, "v1|%d|%d|%d", tenantID, documentID, exp)
return base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
}
// copyFile is the cross-device fallback for os.Rename (EXDEV): copy + fsync.
// The source is removed by the caller.
func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o640)
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
out.Close()
os.Remove(dst)
return err
}
if err := out.Sync(); err != nil {
out.Close()
os.Remove(dst)
return err
}
return out.Close()
}
+204
View File
@@ -0,0 +1,204 @@
package objectstore
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"time"
"archivdms/config"
)
func newTestStore(t *testing.T) *LocalStore {
t.Helper()
l, err := NewLocalStore(config.StorageConfig{BasePath: t.TempDir()}, "test-master-secret", "https://dms.example.test")
if err != nil {
t.Fatalf("NewLocalStore: %v", err)
}
return l
}
// stage writes a scratch file and returns its path plus content hash.
func stage(t *testing.T, content string) (string, string) {
t.Helper()
p := filepath.Join(t.TempDir(), "scratch.pdf")
if err := os.WriteFile(p, []byte(content), 0o640); err != nil {
t.Fatalf("write scratch: %v", err)
}
sum := sha256.Sum256([]byte(content))
return p, hex.EncodeToString(sum[:])
}
// AK 1 + Prüfung 1: Round-Trip Archive -> Open, WORM path scheme and 0440.
func TestArchiveOpenRoundTrip(t *testing.T) {
l := newTestStore(t)
ctx := context.Background()
src, hash := stage(t, "hello worm")
at := time.Date(2026, 3, 7, 10, 0, 0, 0, time.UTC)
dst, err := l.Archive(ctx, 42, src, ".pdf", hash, at)
if err != nil {
t.Fatalf("Archive: %v", err)
}
want := filepath.Join(l.cfg.StorePath(), "42", "2026", "03", hash+".pdf")
if dst != want {
t.Fatalf("path scheme changed: got %q want %q", dst, want)
}
fi, err := os.Stat(dst)
if err != nil {
t.Fatalf("stat archived: %v", err)
}
if fi.Mode().Perm() != 0o440 {
t.Fatalf("WORM permissions: got %o want 0440", fi.Mode().Perm())
}
if _, err := os.Stat(src); !os.IsNotExist(err) {
t.Fatalf("scratch file not consumed")
}
f, err := l.Open(ctx, 42, dst)
if err != nil {
t.Fatalf("Open: %v", err)
}
defer f.Close()
got, _ := io.ReadAll(f)
if string(got) != "hello worm" {
t.Fatalf("round-trip content mismatch: %q", got)
}
// Duplicate archival of the same content is rejected.
src2, _ := stage(t, "hello worm")
if _, err := l.Archive(ctx, 42, src2, ".pdf", hash, at); !errors.Is(err, ErrObjectExists) {
t.Fatalf("duplicate: got %v want ErrObjectExists", err)
}
}
// AK 1: cross-tenant access is refused even with a valid path.
func TestOpenForeignTenantRejected(t *testing.T) {
l := newTestStore(t)
ctx := context.Background()
src, hash := stage(t, "tenant one")
dst, err := l.Archive(ctx, 1, src, ".pdf", hash, time.Now())
if err != nil {
t.Fatalf("Archive: %v", err)
}
if _, err := l.Open(ctx, 2, dst); !errors.Is(err, ErrOutsideTenant) {
t.Fatalf("foreign tenant: got %v want ErrOutsideTenant", err)
}
if _, err := l.Open(ctx, 1, filepath.Join(filepath.Dir(dst), "..", "..", "..", "2", "x.pdf")); !errors.Is(err, ErrOutsideTenant) {
t.Fatalf("traversal: got %v want ErrOutsideTenant", err)
}
}
// Prüfung 3: missing object yields a clear, typed error.
func TestMissingObjectErrors(t *testing.T) {
l := newTestStore(t)
ctx := context.Background()
missing := filepath.Join(l.cfg.StorePath(), "7", "2026", "01", "deadbeef.pdf")
if _, err := l.Open(ctx, 7, missing); !errors.Is(err, ErrObjectNotFound) {
t.Fatalf("Open: got %v want ErrObjectNotFound", err)
}
if _, err := l.Stat(ctx, 7, missing); !errors.Is(err, ErrObjectNotFound) {
t.Fatalf("Stat: got %v want ErrObjectNotFound", err)
}
if err := l.Delete(ctx, 7, missing); !errors.Is(err, ErrObjectNotFound) {
t.Fatalf("Delete: got %v want ErrObjectNotFound", err)
}
if _, err := l.Open(ctx, 7, ""); !errors.Is(err, ErrObjectNotFound) {
t.Fatalf("empty path: got %v want ErrObjectNotFound", err)
}
}
func TestDeleteRemovesObject(t *testing.T) {
l := newTestStore(t)
ctx := context.Background()
src, hash := stage(t, "to be deleted")
dst, err := l.Archive(ctx, 5, src, ".pdf", hash, time.Now())
if err != nil {
t.Fatalf("Archive: %v", err)
}
if err := l.Delete(ctx, 5, dst); err != nil {
t.Fatalf("Delete: %v", err)
}
if _, err := os.Stat(dst); !os.IsNotExist(err) {
t.Fatalf("object still present after delete")
}
}
// AK 2 + Prüfung 2: signed URLs verify while valid and are refused afterwards.
func TestSignedURLLifecycle(t *testing.T) {
l := newTestStore(t)
link, err := l.SignedURL(3, 99, time.Minute)
if err != nil {
t.Fatalf("SignedURL: %v", err)
}
if !strings.HasPrefix(link, "https://dms.example.test"+SignedPath+"?") {
t.Fatalf("unexpected link: %s", link)
}
u, err := url.Parse(link)
if err != nil {
t.Fatalf("parse link: %v", err)
}
ref, err := l.VerifySignedURL(u.Query(), time.Now())
if err != nil {
t.Fatalf("verify: %v", err)
}
if ref.TenantID != 3 || ref.DocumentID != 99 {
t.Fatalf("payload mismatch: %+v", ref)
}
// Expired.
if _, err := l.VerifySignedURL(u.Query(), time.Now().Add(2*time.Minute)); !errors.Is(err, ErrSignatureExpired) {
t.Fatalf("expired: got %v want ErrSignatureExpired", err)
}
// Tampered document id.
q := u.Query()
q.Set("d", "100")
if _, err := l.VerifySignedURL(q, time.Now()); !errors.Is(err, ErrSignatureInvalid) {
t.Fatalf("tampered: got %v want ErrSignatureInvalid", err)
}
// Foreign key material.
other, err := NewLocalStore(config.StorageConfig{BasePath: t.TempDir()}, "different-secret", "")
if err != nil {
t.Fatalf("NewLocalStore: %v", err)
}
if _, err := other.VerifySignedURL(u.Query(), time.Now()); !errors.Is(err, ErrSignatureInvalid) {
t.Fatalf("foreign key: got %v want ErrSignatureInvalid", err)
}
// Missing parameters.
if _, err := l.VerifySignedURL(url.Values{}, time.Now()); !errors.Is(err, ErrSignatureInvalid) {
t.Fatalf("empty query: got %v want ErrSignatureInvalid", err)
}
}
// AK 2: ttl <= 0 falls back to the configured default validity.
func TestSignedURLDefaultTTL(t *testing.T) {
l, err := NewLocalStore(config.StorageConfig{BasePath: t.TempDir(), SignedURLTTLMinutes: 5}, "s", "")
if err != nil {
t.Fatalf("NewLocalStore: %v", err)
}
link, err := l.SignedURL(1, 1, 0)
if err != nil {
t.Fatalf("SignedURL: %v", err)
}
u, _ := url.Parse(link)
if strings.HasPrefix(link, "http") {
t.Fatalf("empty baseURL must yield a relative link: %s", link)
}
if _, err := l.VerifySignedURL(u.Query(), time.Now().Add(4*time.Minute)); err != nil {
t.Fatalf("within default ttl: %v", err)
}
if _, err := l.VerifySignedURL(u.Query(), time.Now().Add(6*time.Minute)); !errors.Is(err, ErrSignatureExpired) {
t.Fatalf("past default ttl: got %v want ErrSignatureExpired", err)
}
}
+105
View File
@@ -0,0 +1,105 @@
// Package objectstore puts the existing local WORM document storage behind a
// small Go interface (FDN-03). It is a pure abstraction layer: the on-disk
// layout, the chmod 0440 WORM lock and the SHA-256 content addressing are
// exactly the ones the upload pipeline has always used — nothing about the
// path scheme or the archival semantics changes here.
//
// # Pfadschema (bestehend, NICHT verändert)
//
// All paths are rooted at config.Storage.BasePath (default /var/lib/archivdms):
//
// <BasePath>/inbox/<tenant_id>/<random>.<ext> raw upload, scratch, writable (0640)
// <BasePath>/store/<tenant_id>/<yyyy>/<mm>/<sha256>.<ext> finished archive, WORM (0440)
// <BasePath>/ocr-tmp/<random>/ OCR scratch, removed after use
// <BasePath>/thumbnails/<tenant_id>/<sha256>.png regenerable preview, not WORM
//
// Properties of the store/ layer that callers may rely on:
//
// - Tenant separation is the FIRST path segment: every object of a tenant
// lives below store/<tenant_id>/ and nowhere else. Open/Stat/Delete
// therefore verify that the given path really is inside that tenant's
// subtree (containment check) — a stored path from a foreign tenant is
// rejected with ErrOutsideTenant instead of being read.
// - <yyyy>/<mm> is derived from the archival (upload) time, not from the
// recognised Belegdatum: after the WORM move a file is never moved again.
// - The file name is the lowercase hex SHA-256 of the file content plus the
// original extension. Content addressing gives byte-identical re-uploads
// the same path, which is the filesystem half of the duplicate protection
// (the DB unique index on (tenant_id, content_hash) is the other half).
// - Archived files are chmod 0440. The directory stays writable for the
// service user, so a legally confirmed deletion (after retain_until) can
// still unlink the file — no code path ever overwrites an archived file.
// - Nothing is encrypted or container-wrapped: every object is readable with
// plain OS tools, deliberately unlike a closed vendor archive.
//
// Deliberately NO S3/object-storage driver: the WORM/GoBD guarantee rests on
// POSIX file permissions (0440) which an object store cannot provide in the
// same way. The local driver is and stays the only implementation.
package objectstore
import (
"context"
"errors"
"io"
"net/url"
"os"
"time"
)
// Storage-level errors. Callers map these onto HTTP status codes / domain
// errors (e.g. ErrObjectExists -> storage.ErrDuplicateContentHash).
var (
// ErrObjectExists is returned by Archive when the target WORM path is
// already taken, i.e. the identical content is already archived.
ErrObjectExists = errors.New("objectstore: object already exists")
// ErrObjectNotFound is returned by Open/Stat/Delete when the object does
// not exist on disk.
ErrObjectNotFound = errors.New("objectstore: object not found")
// ErrOutsideTenant is returned when a storage path does not resolve into
// the requesting tenant's store subtree (IDOR / path-traversal guard).
ErrOutsideTenant = errors.New("objectstore: path outside tenant store")
// ErrSignatureInvalid is returned when a signed URL is malformed or its
// HMAC does not verify.
ErrSignatureInvalid = errors.New("objectstore: signature invalid")
// ErrSignatureExpired is returned when a signed URL's expiry has passed.
ErrSignatureExpired = errors.New("objectstore: signature expired")
)
// SignedRef is the payload carried by a signed download URL: which document of
// which tenant may be downloaded, and until when.
type SignedRef struct {
TenantID int64
DocumentID int64
ExpiresAt time.Time
}
// Store is the document blob storage abstraction. Every method is
// tenant-scoped; there is intentionally no "list everything" call.
type Store interface {
// Archive moves an already-hashed scratch file (inbox or split part) into
// the tenant's WORM store and locks it with chmod 0440. It returns the
// final storage path. On success the caller no longer owns srcPath; on
// failure srcPath is removed. Returns ErrObjectExists when the content is
// already archived (duplicate).
Archive(ctx context.Context, tenantID int64, srcPath, ext, contentHash string, at time.Time) (string, error)
// Open opens an archived object read-only after verifying that
// storagePath belongs to tenantID.
Open(ctx context.Context, tenantID int64, storagePath string) (io.ReadSeekCloser, error)
// Stat reports metadata of an archived object (tenant-checked).
Stat(ctx context.Context, tenantID int64, storagePath string) (os.FileInfo, error)
// Delete unlinks an archived object (tenant-checked). Only ever called
// after a confirmed, retention-cleared deletion request; a missing file is
// reported as ErrObjectNotFound.
Delete(ctx context.Context, tenantID int64, storagePath string) error
// SignedURL builds a time-limited, HMAC-signed download URL for a
// document. ttl <= 0 uses the configured default validity.
SignedURL(tenantID, documentID int64, ttl time.Duration) (string, error)
// VerifySignedURL validates the query parameters of a signed URL against
// the signing key and the current time.
VerifySignedURL(q url.Values, now time.Time) (SignedRef, error)
}
@@ -0,0 +1,27 @@
-- 024_ocr_words.down.sql
-- Rollback documentation for 024_ocr_words.sql.
-- Documentation only — archivdms has no migration runner; run manually via
-- psql after deploying a binary whose initSchema no longer creates the table.
--
-- PRECONDITION: Store.initOCRWordsSchema must be removed from
-- Store.initSchema (internal/storage/documents.go) and every caller of
-- ReplaceOCRWords / the word-box read path must be gone first — otherwise
-- the next process start recreates the table.
--
-- DATA LOSS: ocr_words is a DERIVED index over each document's OCR run, not
-- an original record. Dropping it loses no GoBD-relevant data; the word boxes
-- are fully regenerable by re-running OCR
-- (`archivdms documents reprocess-all`). Only the highlight/overlay feature
-- degrades until then.
--
-- WORM: no archived file in store/ is touched; the FK to documents is only
-- consumed here (ON DELETE CASCADE), never the other way round, so dropping
-- this table cannot cascade into documents.
BEGIN;
DROP INDEX IF EXISTS idx_ocr_words_word_text;
DROP INDEX IF EXISTS idx_ocr_words_document;
DROP TABLE IF EXISTS ocr_words;
COMMIT;
@@ -0,0 +1,25 @@
-- 025_document_date_score.down.sql
-- Rollback documentation for 025_document_date_score.sql.
-- Documentation only — archivdms has no migration runner; run manually via
-- psql after deploying a binary whose initSchema no longer adds the column.
--
-- PRECONDITION: the ALTER TABLE ... ADD COLUMN IF NOT EXISTS
-- document_date_score must be removed from Store.initSchema
-- (internal/storage/documents.go) first, and every SELECT/UPDATE naming
-- document_date_score (accounting_pull.go, document date endpoint) must be
-- gone — otherwise the next start recreates the column and running queries
-- fail in between.
--
-- DATA LOSS: drops the per-document confidence values. documents.document_date
-- itself is NOT touched, so no belegdatum is lost — only the quality signal
-- the Buchhaltungs-Pull-Filter (score >= 0.75) uses. Recomputable only by
-- re-scoring the stored ocr_text.
--
-- WORM: this is a metadata column on documents; dropping it does not touch
-- any archived file in store/ and does not affect retain_until.
BEGIN;
ALTER TABLE documents DROP COLUMN IF EXISTS document_date_score;
COMMIT;
@@ -0,0 +1,24 @@
-- 026_accounting_api_keys.down.sql
-- Rollback documentation for 026_accounting_api_keys.sql.
-- Documentation only — archivdms has no migration runner; this file is the
-- reviewed, copy-pasteable SQL an operator runs manually (psql) AFTER
-- deploying a binary whose initSchema no longer creates the object, and
-- inside an explicit transaction.
--
-- PRECONDITION: internal/storage/accounting_api_keys.go
-- (Store.initAccountingAPIKeysSchema) must be removed from storage.New()
-- first — otherwise the next process start recreates the table.
--
-- DATA LOSS: destroys all issued accounting API keys (hashes only, raw keys
-- were never stored). Every buchhaltung integration using the Pull-API loses
-- access and must be re-issued a new key. No GoBD document data is touched:
-- accounting_api_keys holds credentials, not archived records. The audit
-- entries referencing "key:<id>" survive in audit_log but their id becomes
-- unresolvable — accept this only when the whole feature is being withdrawn.
BEGIN;
DROP INDEX IF EXISTS idx_accounting_api_keys_tenant;
DROP TABLE IF EXISTS accounting_api_keys;
COMMIT;
+39
View File
@@ -23,6 +23,45 @@ Migrationstool. Sie dient als:
3. Der SQL-Inhalt muss exakt dem entsprechen, was `initSchema()` (oder das
jeweilige Store-Paket) zur Laufzeit ausführt.
4. Migrationen werden nie verändert oder gelöscht, nur ergänzt.
5. **Zu jeder neuen Migration gehört eine Down-Datei** `NNN_name.down.sql`
(siehe Abschnitt „Rollback-Pfad").
## Rollback-Pfad (`NNN_name.down.sql`)
`initSchema()` ist ausschließlich vorwärtsgerichtet — es gibt bewusst keinen
automatischen Rollback-Runner (kein Migrationstool, kein Zustandstabellen-
Tracking). Für den Ernstfall (fehlerhaftes Release, Rückbau eines Features)
braucht es trotzdem ein *reviewtes* Rückbau-SQL. Deshalb gilt ab FDN-02:
**Jede neue `NNN_name.sql` bekommt eine gleichnamige `NNN_name.down.sql`**
mit dem exakten Rückbau der Vorwärts-Migration. Separate Datei statt
`-- DOWN`-Abschnitt in derselben Datei, weil Regel 4 („Migrationen werden nie
verändert") sonst verletzt würde und weil sich eine Down-Datei fehlerfrei
per `psql -f` einspielen lässt, ohne vorher Abschnitte herauszuschneiden.
Anforderungen an eine Down-Datei:
1. Header-Kommentar mit Bezug auf die Vorwärts-Migration und der zugehörigen
PROJ-/Ticket-Nummer.
2. **Precondition** benennen: welche Go-Stelle (`initSchema`, Store-Datei,
aufrufende Queries) vorher entfernt bzw. deployt sein muss. Sonst legt der
nächste Prozessstart das Objekt sofort wieder an.
3. **Datenverlust explizit benennen** — was ist danach unwiederbringlich weg,
was ist regenerierbar (z.B. abgeleitete Indizes wie `ocr_words`).
4. **WORM/GoBD-Hinweis**: klarstellen, dass kein archiviertes File unter
`store/` und kein `retain_until` berührt wird. Down-SQL darf niemals
Dokument-Nutzdaten oder Aufbewahrungssperren löschen.
5. Idempotent formulieren (`DROP ... IF EXISTS`) und in `BEGIN; ... COMMIT;`
klammern.
Ausführung ist immer **manuell und bewusst** (`psql -f
internal/storage/migrations/NNN_name.down.sql`), nie automatisch beim Start.
Beispiele (rückwirkend ergänzt, dienen als Vorlage):
`024_ocr_words.down.sql`, `025_document_date_score.down.sql`,
`026_accounting_api_keys.down.sql`. Ältere Migrationen (001023) haben keine
Down-Datei — sie beschreiben das etablierte Kernschema, dessen Rückbau kein
realistisches Szenario ist.
## Vorhandene Migrationen
+30
View File
@@ -340,6 +340,36 @@ func (s *Store) ReapStaleJobs(ctx context.Context, timeout time.Duration, maxRet
return len(stale), nil
}
// CountProcessingJobsByStatus liefert die Queue-Länge je Status über ALLE
// Mandanten hinweg. Bewusst ohne tenant_id-Filter: einziger Aufrufer ist der
// betriebsinterne Prometheus-Endpunkt GET /metrics (FDN-08), der nur
// aggregierte Zahlen ohne Mandantenbezug ausgibt — es verlassen keine
// mandantenbezogenen Daten das System. Für mandantenbezogene Auswertungen
// niemals diese Funktion nutzen.
func (s *Store) CountProcessingJobsByStatus(ctx context.Context) (map[string]int64, error) {
rows, err := s.db.Query(ctx, `
SELECT status, count(*) FROM processing_jobs GROUP BY status
`)
if err != nil {
return nil, fmt.Errorf("storage: count processing jobs by status: %w", err)
}
defer rows.Close()
out := map[string]int64{}
for rows.Next() {
var status string
var n int64
if err := rows.Scan(&status, &n); err != nil {
return nil, fmt.Errorf("storage: scan processing job count: %w", err)
}
out[status] = n
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("storage: iterate processing job counts: %w", err)
}
return out, nil
}
// RequeueJob stellt einen Job (typischerweise einen dauerhaft 'failed'
// gelaufenen) wieder in die Queue und setzt retry_count zurück. Wird vom
// späteren manuellen Retry-Endpunkt (Phase 3) genutzt; tenant-scoped.