FDN-02/FDN-03/FDN-07/FDN-08: Migrations-Rollback, Objekt-Storage-Interface, go.sum-Fix, Observability
CI / Backend (go vet, go test -cover) (push) Has been cancelled
CI / Frontend (ESLint, tsc, next build) (push) Has been cancelled

- FDN-02: Rollback-fähige Down-Migrationen (024-026), archivdms seed dev CLI
- FDN-03: internal/objectstore Interface + lokaler WORM-Treiber, signierte Download-URLs
- FDN-07: go.mod/go.sum vervollständigt (fehlender go-ldap/v3-Eintrag), CI-Pipeline (.gitea/workflows/ci.yml, bereits in FDN-01 committet) damit lauffähig
- FDN-08: Request-ID-Middleware, /metrics-Endpoint, Panic-Recovery, Login/Logout/Me technisches Logging inkl. Access-Log je Anfrage
This commit is contained in:
2026-08-11 22:27:52 +02:00
parent 9a24ea29e1
commit 89de794356
30 changed files with 2688 additions and 113 deletions
+4 -4
View File
@@ -190,7 +190,7 @@ func (s *Server) accountingAuth(h http.HandlerFunc) http.HandlerFunc {
tenantID, keyID, err := s.store.ResolveAccountingAPIKey(r.Context(), rawKey)
if err != nil {
if !errors.Is(err, storage.ErrAccountingKeyNotFound) {
s.logger.Error("accounting api key resolve failed", "err", err)
s.reqLog(r.Context()).Error("accounting api key resolve failed", "err", err)
}
// Unknown, revoked and broken keys are indistinguishable.
s.audlog.Log(audit.Entry{
@@ -290,7 +290,7 @@ func (s *Server) handleAccountingListDocuments(w http.ResponseWriter, r *http.Re
writeError(w, http.StatusBadRequest, "invalid cursor")
return
}
s.logger.Error("accounting list failed", "tenant_id", tenantID, "err", err)
s.reqLog(r.Context()).Error("accounting list failed", "tenant_id", tenantID, "err", err)
s.audlog.Log(audit.Entry{
EventType: audit.EventAccountingPull, Username: "accounting_api", IPAddress: s.remoteIP(r),
TenantID: &tenantID, Success: false,
@@ -342,7 +342,7 @@ func (s *Server) handleAccountingDocumentFile(w http.ResponseWriter, r *http.Req
f, err := os.Open(ref.StoragePath())
if err != nil {
s.logger.Error("accounting file open failed", "document_id", ref.DocumentID, "tenant_id", tenantID, "err", err)
s.reqLog(r.Context()).Error("accounting file open failed", "document_id", ref.DocumentID, "tenant_id", tenantID, "err", err)
s.audlog.Log(audit.Entry{
EventType: audit.EventAccountingPull, Username: "accounting_api", IPAddress: s.remoteIP(r),
TenantID: &tenantID, DocumentID: strconv.FormatInt(id, 10), Success: false,
@@ -364,7 +364,7 @@ func (s *Server) handleAccountingDocumentFile(w http.ResponseWriter, r *http.Req
w.Header().Set("Content-Disposition", "attachment; filename=\""+safeDownloadName(ref.Title, ext)+"\"")
w.Header().Set("X-Content-Type-Options", "nosniff")
if _, err := io.Copy(w, f); err != nil {
s.logger.Warn("accounting file stream interrupted", "document_id", ref.DocumentID, "err", err)
s.reqLog(r.Context()).Warn("accounting file stream interrupted", "document_id", ref.DocumentID, "err", err)
}
}
+30 -5
View File
@@ -13,15 +13,22 @@ type loginRequest struct {
}
func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var req loginRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
s.reqLog(ctx).Warn("login: invalid request body",
"remote_ip", s.remoteIP(r), "err", err)
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
ip := s.remoteIP(r)
token, user, err := s.authMgr.LoginFrom(r.Context(), req.Username, req.Password, ip)
token, user, err := s.authMgr.LoginFrom(ctx, req.Username, req.Password, ip)
if err != nil {
// Kein Passwort, keine Fehlerdetails vom Auth-Manager im Klartext:
// nur Benutzername + IP zur Korrelation von Brute-Force-Versuchen.
s.reqLog(ctx).Warn("login failed",
"username", req.Username, "remote_ip", ip, "reason", "invalid_credentials")
s.audlog.Log(audit.Entry{
EventType: audit.EventLogin,
Username: req.Username,
@@ -43,7 +50,14 @@ func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
MaxAge: 8 * 60 * 60,
})
_ = s.users.UpdateLastLogin(user.ID)
if err := s.users.UpdateLastLogin(user.ID); err != nil {
s.reqLog(ctx).Warn("login: last_login update failed",
"user_id", user.ID, "err", err)
}
s.reqLog(ctx).Info("login succeeded",
"user_id", user.ID, "username", user.Username,
"tenant_id", user.TenantID, "remote_ip", ip)
s.audlog.Log(audit.Entry{
EventType: audit.EventLogin,
@@ -57,9 +71,12 @@ func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) handleMe(w http.ResponseWriter, r *http.Request) {
sess := sessionFromCtx(r.Context())
ctx := r.Context()
sess := sessionFromCtx(ctx)
user, err := s.users.GetByID(sess.UserID)
if err != nil {
s.reqLog(ctx).Warn("me: user lookup failed",
"user_id", sess.UserID, "err", err)
writeError(w, http.StatusNotFound, "user not found")
return
}
@@ -67,7 +84,8 @@ func (s *Server) handleMe(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request) {
sess := sessionFromCtx(r.Context())
ctx := r.Context()
sess := sessionFromCtx(ctx)
token := ""
if c, err := r.Cookie(sessionCookieName); err == nil {
token = c.Value
@@ -76,9 +94,16 @@ func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request) {
token = extractBearerToken(r)
}
if token != "" {
_ = s.authMgr.Logout(token)
if err := s.authMgr.Logout(token); err != nil {
s.reqLog(ctx).Warn("logout: session invalidation failed",
"user_id", sess.UserID, "err", err)
}
}
s.reqLog(ctx).Info("logout",
"user_id", sess.UserID, "username", sess.Username,
"tenant_id", sess.TenantID, "remote_ip", s.remoteIP(r))
http.SetCookie(w, &http.Cookie{
Name: sessionCookieName,
Value: "",
+1 -1
View File
@@ -21,7 +21,7 @@ func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) {
stats, err := s.store.GetDashboardStats(r.Context(), *sess.TenantID, sess.UserID)
if err != nil {
s.logger.Error("dashboard stats failed", "err", err)
s.reqLog(r.Context()).Error("dashboard stats failed", "err", err)
writeError(w, http.StatusInternalServerError, "dashboard stats failed")
return
}
@@ -197,7 +197,7 @@ func (s *Server) handleBulkExportDocuments(w http.ResponseWriter, r *http.Reques
if err != nil {
// A single unreadable document must not kill the archive — unless the
// ZIP writer itself failed, which we detect on Close below.
s.logger.Warn("bulk export: document skipped", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(r.Context()).Warn("bulk export: document skipped", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
skipped = append(skipped, fmt.Sprintf("%d: %v", doc.ID, err))
continue
}
@@ -225,7 +225,7 @@ func (s *Server) handleBulkExportDocuments(w http.ResponseWriter, r *http.Reques
detail := fmt.Sprintf("zip_bulk_export: exported=%d skipped=%d", exported, len(skipped))
if streamErr != nil {
// Headers are already out — audit the partial export, no HTTP error.
s.logger.Warn("bulk document export stream failed", "tenant_id", tenantID, "err", streamErr)
s.reqLog(r.Context()).Warn("bulk document export stream failed", "tenant_id", tenantID, "err", streamErr)
s.audlog.Log(audit.Entry{
EventType: audit.EventDocumentBulkExport, Username: sess.Username, TenantID: sess.TenantID,
Success: false, Detail: detail + " stream_failed: " + streamErr.Error(),
+2 -2
View File
@@ -143,7 +143,7 @@ func (s *Server) handleExportDocument(w http.ResponseWriter, r *http.Request) {
f, err := os.Open(doc.StoragePath)
if err != nil {
s.logger.Error("export: document file open failed", "document_id", doc.ID, "tenant_id", *sess.TenantID, "err", err)
s.reqLog(r.Context()).Error("export: document file open failed", "document_id", doc.ID, "tenant_id", *sess.TenantID, "err", err)
fail(http.StatusInternalServerError, "file unavailable", "file_open_failed: "+err.Error())
return
}
@@ -212,7 +212,7 @@ func (s *Server) handleExportDocument(w http.ResponseWriter, r *http.Request) {
if streamErr != nil {
// Headers are already out — log + audit the partial export, no HTTP error.
s.logger.Warn("document export stream failed", "document_id", doc.ID, "err", streamErr)
s.reqLog(r.Context()).Warn("document export stream failed", "document_id", doc.ID, "err", streamErr)
s.audlog.Log(audit.Entry{
EventType: audit.EventDocumentExport, Username: sess.Username, TenantID: sess.TenantID,
DocumentID: idStr, Success: false, Detail: "stream_failed: " + streamErr.Error(),
+43 -84
View File
@@ -21,6 +21,7 @@ import (
"archivdms/internal/auth"
"archivdms/internal/dateformat"
"archivdms/internal/matching"
"archivdms/internal/objectstore"
"archivdms/internal/ocr"
"archivdms/internal/storage"
"archivdms/internal/userstore"
@@ -138,10 +139,12 @@ func (s *Server) handleGetDocumentFile(w http.ResponseWriter, r *http.Request) {
return
}
f, err := os.Open(doc.StoragePath)
// Read through the storage abstraction: it re-checks that the stored path
// really lies inside this tenant's store subtree before opening.
f, err := s.objects.Open(r.Context(), *sess.TenantID, doc.StoragePath)
if err != nil {
// WORM store should always hold the file, but never trust the disk.
s.logger.Error("document file open failed", "document_id", doc.ID, "tenant_id", *sess.TenantID, "storage_path", doc.StoragePath, "err", err)
s.reqLog(r.Context()).Error("document file open failed", "document_id", doc.ID, "tenant_id", *sess.TenantID, "storage_path", doc.StoragePath, "err", err)
writeError(w, http.StatusInternalServerError, "file unavailable")
return
}
@@ -152,7 +155,7 @@ func (s *Server) handleGetDocumentFile(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Disposition", "inline; filename=\""+safeDownloadName(doc.Title, ext)+"\"")
w.Header().Set("X-Content-Type-Options", "nosniff")
if _, err := io.Copy(w, f); err != nil {
s.logger.Warn("document file stream interrupted", "document_id", doc.ID, "err", err)
s.reqLog(r.Context()).Warn("document file stream interrupted", "document_id", doc.ID, "err", err)
}
}
@@ -209,7 +212,7 @@ func (s *Server) handleGetDocumentThumbnail(w http.ResponseWriter, r *http.Reque
w.Header().Set("Cache-Control", "private, max-age=86400")
w.Header().Set("X-Content-Type-Options", "nosniff")
if _, err := io.Copy(w, f); err != nil {
s.logger.Warn("thumbnail stream interrupted", "document_id", doc.ID, "err", err)
s.reqLog(r.Context()).Warn("thumbnail stream interrupted", "document_id", doc.ID, "err", err)
}
}
@@ -453,7 +456,7 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
mimeType := detectMimeType("", ext, doc.StoragePath)
result, err := s.ocr.Extract(ctx, doc.StoragePath, mimeType)
if err != nil {
s.logger.Warn("reprocess ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID, "storage_path", doc.StoragePath, "err", err)
s.reqLog(ctx).Warn("reprocess ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID, "storage_path", doc.StoragePath, "err", err)
s.audlog.Log(audit.Entry{EventType: audit.EventDocumentReprocessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "ocr_failed: " + err.Error()})
return nil, fmt.Errorf("reprocess ocr extract: %w", err)
}
@@ -466,7 +469,7 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
// alongside the new ones — best-effort, never fails the reprocess since
// ocr_text is already the authoritative persisted result.
if err := s.store.ReplaceOCRWords(ctx, id, ocrWordsFromResult(id, result.Words)); err != nil {
s.logger.Warn("reprocess replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("reprocess replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
}
if err := s.store.UpdateDocumentOCRText(ctx, id, tenantID, ocrText); err != nil {
@@ -491,7 +494,7 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
prefix, dateLayout := s.tenantScanTitleParams(ctx, tenantID)
if newTitle := titleFromOCRText(ocrText, prefix, dateLayout); newTitle != "" && newTitle != doc.Title {
if err := s.store.UpdateDocumentTitleAuto(ctx, id, tenantID, newTitle); err != nil {
s.logger.Warn("reprocess title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("reprocess title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
} else {
doc.Title = newTitle
}
@@ -513,7 +516,7 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
}
if !sameDate(datePtr, doc.DocumentDate) {
if err := s.store.UpdateDocumentDate(ctx, id, tenantID, datePtr, scorePtr); err != nil {
s.logger.Warn("reprocess document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("reprocess document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
} else {
doc.DocumentDate = datePtr
doc.DocumentDateScore = scorePtr
@@ -524,12 +527,12 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
// Additive auto-assignment (only fills unset doc_type/correspondent, only
// attaches tags) — best-effort, never fails the request.
if assignWarn := s.autoAssignTaxonomy(ctx, tenantID, doc, barcodes, ocrText); assignWarn != "" {
s.logger.Info("reprocess auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn)
s.reqLog(ctx).Info("reprocess auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn)
}
// Re-evaluate on_upload workflows — best-effort, never fails the request.
if err := s.store.RunWorkflowsForDocument(ctx, tenantID, doc, storage.WorkflowTriggerOnUpload); err != nil {
s.logger.Warn("reprocess workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("reprocess workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
}
// Re-read so the response reflects any doc_type/correspondent set by
@@ -726,11 +729,11 @@ func (s *Server) generateThumbnailBestEffort(ctx context.Context, tenantID, docu
}
thumbPath := filepath.Join(s.storageCfg.ThumbnailPath(), strconv.FormatInt(tenantID, 10), contentHash+".png")
if err := s.thumbs.Generate(ctx, storagePath, mimeType, thumbPath); err != nil {
s.logger.Warn("eager thumbnail generation failed", "document_id", documentID, "tenant_id", tenantID, "mime_type", mimeType, "err", err)
s.reqLog(ctx).Warn("eager thumbnail generation failed", "document_id", documentID, "tenant_id", tenantID, "mime_type", mimeType, "err", err)
return
}
if err := s.store.SetDocumentHasThumbnail(ctx, documentID, tenantID, true); err != nil {
s.logger.Warn("has_thumbnail flag update failed", "document_id", documentID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("has_thumbnail flag update failed", "document_id", documentID, "tenant_id", tenantID, "err", err)
}
}
@@ -753,7 +756,7 @@ func (s *Server) handleUploadDocument(w http.ResponseWriter, r *http.Request) {
tenantID := *sess.TenantID
maxBytes := int64(s.storageCfg.ResolvedMaxUploadSizeMB()) * 1024 * 1024
s.logger.Info("upload request received", "username", sess.Username, "tenant_id", tenantID,
s.reqLog(r.Context()).Info("upload request received", "username", sess.Username, "tenant_id", tenantID,
"content_length", r.ContentLength, "max_bytes", maxBytes, "remote_addr", r.RemoteAddr)
r.Body = http.MaxBytesReader(w, r.Body, maxBytes)
@@ -763,7 +766,7 @@ func (s *Server) handleUploadDocument(w http.ResponseWriter, r *http.Request) {
// hitting MaxBytesReader — both close the connection before any
// later log/audit call would otherwise run, which previously left
// zero trace of the failure in the backend logs (see DEVLOG 2026-07-15).
s.logger.Warn("upload parse failed", "username", sess.Username, "tenant_id", tenantID,
s.reqLog(r.Context()).Warn("upload parse failed", "username", sess.Username, "tenant_id", tenantID,
"content_length", r.ContentLength, "max_bytes", maxBytes, "err", err)
s.audlog.Log(audit.Entry{
EventType: audit.EventDocumentCreate, Username: sess.Username, TenantID: sess.TenantID,
@@ -956,41 +959,19 @@ func (s *Server) archiveStagedFile(
}
pathDate := time.Now()
// 4. Build the WORM target path store/<tenant>/<yyyy>/<mm>/<hash>.<ext>.
storeDir := filepath.Join(s.storageCfg.StorePath(), strconv.FormatInt(tenantID, 10),
fmt.Sprintf("%04d", pathDate.Year()), fmt.Sprintf("%02d", pathDate.Month()))
if err := os.MkdirAll(storeDir, 0o750); err != nil {
os.Remove(inboxPath)
return nil, "", fmt.Errorf("create store dir: %w", err)
}
storePath := filepath.Join(storeDir, contentHash+ext)
// 5. Collision check: identical hash already stored -> reject as
// duplicate before touching the DB (filesystem-level half of the
// duplicate protection; the DB unique index is the other half).
if _, err := os.Stat(storePath); err == nil {
os.Remove(inboxPath)
return nil, "", storage.ErrDuplicateContentHash
} else if !os.IsNotExist(err) {
os.Remove(inboxPath)
return nil, "", fmt.Errorf("stat store path: %w", err)
}
// 6. Move inbox -> store. Prefer atomic rename; fall back to copy+remove
// if inbox/store ever end up on different filesystems/mounts.
if err := os.Rename(inboxPath, storePath); err != nil {
if copyErr := copyFile(inboxPath, storePath); copyErr != nil {
os.Remove(inboxPath)
return nil, "", fmt.Errorf("move file to store: rename failed (%v), copy fallback failed: %w", err, copyErr)
// 4./5./6./7. WORM archival — delegated unchanged to the storage
// abstraction (internal/objectstore): build the target path
// store/<tenant>/<yyyy>/<mm>/<hash>.<ext>, reject an already-stored hash as
// duplicate (filesystem half of the duplicate protection; the DB unique
// index is the other half), move inbox -> store (rename, copy+remove
// fallback across mounts) and finally chmod 0440. Path scheme and
// semantics are identical to the previous inline implementation.
storePath, err := s.objects.Archive(ctx, tenantID, inboxPath, ext, contentHash, pathDate)
if err != nil {
if errors.Is(err, objectstore.ErrObjectExists) {
return nil, "", storage.ErrDuplicateContentHash
}
os.Remove(inboxPath)
}
// 7. WORM lock: read-only, no write access for anyone once archived. This is
// the ONLY chmod and it happens exactly once, after the file reaches its
// final path — the file is never moved or renamed again afterwards.
if err := os.Chmod(storePath, 0o440); err != nil {
return nil, "", fmt.Errorf("chmod store file: %w", err)
return nil, "", err
}
// 8. Record the document AND its processing job in ONE transaction. If no
@@ -1018,7 +999,7 @@ func (s *Server) archiveStagedFile(
if err != nil {
return nil, "", err
}
s.logger.Info("document staged, processing job queued",
s.reqLog(ctx).Info("document staged, processing job queued",
"document_id", doc.ID, "tenant_id", tenantID, "job_id", job.ID, "derive_title", deriveTitle)
// 7b. Eager thumbnail generation: same pipeline/cache path as the lazy
@@ -1084,7 +1065,7 @@ func (s *Server) trySplitStagedUpload(
res, split, err := s.pagesplitter.Split(ctx, inboxPath)
if err != nil {
s.logger.Warn("separator-page split failed, archiving document unsplit",
s.reqLog(ctx).Warn("separator-page split failed, archiving document unsplit",
"tenant_id", tenantID, "filename", filename, "err", err)
s.audlog.Log(audit.Entry{
EventType: audit.EventDocumentSplit, Username: splitActor, TenantID: &tenantID, Success: false,
@@ -1167,7 +1148,7 @@ func (s *Server) trySplitStagedUpload(
EventType: audit.EventDocumentSplit, Username: splitActor, TenantID: &tenantID,
DocumentID: strconv.FormatInt(docs[0].ID, 10), Success: true, Detail: detail,
})
s.logger.Info("upload split at barcode separator pages",
s.reqLog(ctx).Info("upload split at barcode separator pages",
"tenant_id", tenantID, "filename", filename, "pages", res.PageCount,
"separator_pages", res.SeparatorPages, "parts", len(res.Parts),
"documents_created", len(docs), "duplicates_skipped", dupCount)
@@ -1238,7 +1219,7 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
mimeType := detectMimeType("", ext, doc.StoragePath)
result, err := s.ocr.Extract(ctx, doc.StoragePath, mimeType)
if err != nil {
s.logger.Warn("jobqueue ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID,
s.reqLog(ctx).Warn("jobqueue ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID,
"storage_path", doc.StoragePath, "resolved_mime", mimeType, "err", err)
s.audlog.Log(audit.Entry{EventType: audit.EventDocumentProcessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "ocr_failed: " + err.Error()})
return fmt.Errorf("jobqueue ocr extract: %w", err)
@@ -1251,7 +1232,7 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
// retry/backoff) never accumulates duplicates. Best-effort, never fails
// the job since ocr_text is already the authoritative persisted result.
if err := s.store.ReplaceOCRWords(ctx, documentID, ocrWordsFromResult(documentID, result.Words)); err != nil {
s.logger.Warn("jobqueue replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("jobqueue replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
}
if err := s.store.UpdateDocumentOCRText(ctx, documentID, tenantID, ocrText); err != nil {
@@ -1267,7 +1248,7 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
prefix, dateLayout := s.tenantScanTitleParams(ctx, tenantID)
if newTitle := titleFromOCRText(ocrText, prefix, dateLayout); newTitle != "" && newTitle != doc.Title {
if err := s.store.UpdateDocumentTitleAuto(ctx, documentID, tenantID, newTitle); err != nil {
s.logger.Warn("jobqueue title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("jobqueue title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
} else {
doc.Title = newTitle
}
@@ -1286,7 +1267,7 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
}
if !sameDate(datePtr, doc.DocumentDate) {
if err := s.store.UpdateDocumentDate(ctx, documentID, tenantID, datePtr, scorePtr); err != nil {
s.logger.Warn("jobqueue document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("jobqueue document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
} else {
doc.DocumentDate = datePtr
doc.DocumentDateScore = scorePtr
@@ -1295,11 +1276,11 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
}
if assignWarn := s.autoAssignTaxonomy(ctx, tenantID, doc, barcodes, ocrText); assignWarn != "" {
s.logger.Info("jobqueue auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn)
s.reqLog(ctx).Info("jobqueue auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn)
}
if err := s.store.RunWorkflowsForDocument(ctx, tenantID, doc, storage.WorkflowTriggerOnUpload); err != nil {
s.logger.Warn("jobqueue workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
s.reqLog(ctx).Warn("jobqueue workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
}
// Final index sync: auto-assignment/workflows may have re-indexed already,
@@ -1323,7 +1304,7 @@ func (s *Server) autoAssignTaxonomy(ctx context.Context, tenantID int64, doc *st
// Nachvollziehbarkeit even when nothing matched.
if len(barcodes) > 0 {
if err := s.store.SetDocumentBarcodeValues(ctx, doc.ID, tenantID, barcodes); err != nil {
s.logger.Warn("failed to store barcode values", "document_id", doc.ID, "err", err)
s.reqLog(ctx).Warn("failed to store barcode values", "document_id", doc.ID, "err", err)
}
}
@@ -1371,7 +1352,7 @@ func (s *Server) autoAssignTaxonomy(ctx context.Context, tenantID int64, doc *st
for _, kind := range taxonomyKinds {
entities, err := s.store.ListActiveMatchers(ctx, kind, tenantID)
if err != nil {
s.logger.Warn("failed to list active matchers", "kind", kind, "err", err)
s.reqLog(ctx).Warn("failed to list active matchers", "kind", kind, "err", err)
continue
}
for _, entity := range entities {
@@ -1537,31 +1518,9 @@ func randomUploadID() string {
return hex.EncodeToString(b)
}
// copyFile is the cross-device fallback for os.Rename (EXDEV): copy + fsync
// + remove the source.
func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o640)
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
out.Close()
os.Remove(dst)
return err
}
if err := out.Sync(); err != nil {
out.Close()
os.Remove(dst)
return err
}
return out.Close()
}
// The cross-device copy fallback for the WORM move lives in
// internal/objectstore (copyFile there) since FDN-03 moved the archival step
// behind the storage abstraction.
// ocrSupportedMimeTypes is the whitelist of MIME types the OCR pipeline
// (internal/ocr.Extract) actually dispatches on. A declared Content-Type is
+155
View File
@@ -0,0 +1,155 @@
// FDN-08 — /metrics im Prometheus-Textformat, ohne Fremdabhängigkeit.
//
// Zugriff: bewusst OHNE Login (Scrape-Clients haben keine Session), dafür
// IP-beschränkt. Default ist loopback-only; weitere Scraper werden über
// config api.metrics_allowed_ips (IP oder CIDR) freigeschaltet. Es werden
// ausschließlich aggregierte Zähler ausgegeben — keine Tenant-Daten, keine
// Pfadsegmente mit IDs oder Tokens (siehe normalizeRoute).
package api
import (
"context"
"fmt"
"net"
"net/http"
"runtime"
"sort"
"strings"
"time"
)
// handleMetrics rendert die Registry im Prometheus-Textformat.
func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) {
if !s.metricsAllowed(r) {
writeError(w, http.StatusForbidden, "metrics endpoint not allowed from this address")
return
}
routes, inFlight, panics := s.metrics.snapshot()
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
w.WriteHeader(http.StatusOK)
var b strings.Builder
fmt.Fprintf(&b, "# HELP archivdms_build_info Statische Build-Information.\n")
fmt.Fprintf(&b, "# TYPE archivdms_build_info gauge\n")
fmt.Fprintf(&b, "archivdms_build_info{version=\"%s\"} 1\n", escapeLabel(s.appVersion))
fmt.Fprintf(&b, "# HELP archivdms_uptime_seconds Laufzeit des Prozesses in Sekunden.\n")
fmt.Fprintf(&b, "# TYPE archivdms_uptime_seconds gauge\n")
fmt.Fprintf(&b, "archivdms_uptime_seconds %.3f\n", time.Since(s.startTime).Seconds())
fmt.Fprintf(&b, "# HELP archivdms_goroutines Aktuelle Anzahl Goroutinen.\n")
fmt.Fprintf(&b, "# TYPE archivdms_goroutines gauge\n")
fmt.Fprintf(&b, "archivdms_goroutines %d\n", runtime.NumGoroutine())
fmt.Fprintf(&b, "# HELP archivdms_http_requests_in_flight Aktuell laufende HTTP-Anfragen.\n")
fmt.Fprintf(&b, "# TYPE archivdms_http_requests_in_flight gauge\n")
fmt.Fprintf(&b, "archivdms_http_requests_in_flight %d\n", inFlight)
fmt.Fprintf(&b, "# HELP archivdms_panics_total Zentral abgefangene Panics (unbehandelte Fehler).\n")
fmt.Fprintf(&b, "# TYPE archivdms_panics_total counter\n")
fmt.Fprintf(&b, "archivdms_panics_total %d\n", panics)
// Requests + Latenz je Route/Status.
keys := make([]routeKey, 0, len(routes))
for k := range routes {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
if keys[i].route != keys[j].route {
return keys[i].route < keys[j].route
}
if keys[i].method != keys[j].method {
return keys[i].method < keys[j].method
}
return keys[i].status < keys[j].status
})
fmt.Fprintf(&b, "# HELP archivdms_http_requests_total Anzahl HTTP-Anfragen je Route und Status.\n")
fmt.Fprintf(&b, "# TYPE archivdms_http_requests_total counter\n")
for _, k := range keys {
st := routes[k]
fmt.Fprintf(&b, "archivdms_http_requests_total{method=\"%s\",route=\"%s\",status=\"%d\"} %d\n",
escapeLabel(k.method), escapeLabel(k.route), k.status, st.count)
}
fmt.Fprintf(&b, "# HELP archivdms_http_request_duration_seconds Latenz der HTTP-Anfragen.\n")
fmt.Fprintf(&b, "# TYPE archivdms_http_request_duration_seconds histogram\n")
for _, k := range keys {
st := routes[k]
labels := fmt.Sprintf("method=\"%s\",route=\"%s\",status=\"%d\"",
escapeLabel(k.method), escapeLabel(k.route), k.status)
for i, ub := range latencyBuckets {
fmt.Fprintf(&b, "archivdms_http_request_duration_seconds_bucket{%s,le=\"%g\"} %d\n",
labels, ub, st.bucketCount[i])
}
fmt.Fprintf(&b, "archivdms_http_request_duration_seconds_bucket{%s,le=\"+Inf\"} %d\n", labels, st.count)
fmt.Fprintf(&b, "archivdms_http_request_duration_seconds_sum{%s} %.6f\n", labels, st.sumSeconds)
fmt.Fprintf(&b, "archivdms_http_request_duration_seconds_count{%s} %d\n", labels, st.count)
}
// Queue-Länge (Akzeptanzkriterium 2). Fehler hier dürfen den Scrape nicht
// scheitern lassen — dann fehlt die Metrik einfach für diesen Durchlauf.
if s.store != nil {
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
defer cancel()
counts, err := s.store.CountProcessingJobsByStatus(ctx)
if err != nil {
s.reqLog(r.Context()).Warn("metrics: queue length query failed", "err", err)
} else {
fmt.Fprintf(&b, "# HELP archivdms_processing_jobs Länge der Verarbeitungswarteschlange je Status.\n")
fmt.Fprintf(&b, "# TYPE archivdms_processing_jobs gauge\n")
statuses := make([]string, 0, len(counts))
for st := range counts {
statuses = append(statuses, st)
}
sort.Strings(statuses)
for _, st := range statuses {
fmt.Fprintf(&b, "archivdms_processing_jobs{status=\"%s\"} %d\n", escapeLabel(st), counts[st])
}
}
}
w.Write([]byte(b.String()))
}
// metricsAllowed prüft die Herkunft des Scrape-Requests: loopback immer,
// sonst nur konfigurierte IPs/CIDRs (config api.metrics_allowed_ips).
func (s *Server) metricsAllowed(r *http.Request) bool {
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
host = r.RemoteAddr
}
ip := net.ParseIP(strings.TrimSpace(host))
if ip == nil {
return false
}
if ip.IsLoopback() {
return true
}
for _, entry := range s.cfg.MetricsAllowedIPs {
entry = strings.TrimSpace(entry)
if entry == "" {
continue
}
if strings.Contains(entry, "/") {
if _, cidr, err := net.ParseCIDR(entry); err == nil && cidr.Contains(ip) {
return true
}
continue
}
if parsed := net.ParseIP(entry); parsed != nil && parsed.Equal(ip) {
return true
}
}
return false
}
// escapeLabel entschärft Anführungszeichen/Backslashes/Zeilenumbrüche in
// Prometheus-Labelwerten.
func escapeLabel(v string) string {
r := strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`)
return r.Replace(v)
}
+404
View File
@@ -0,0 +1,404 @@
// FDN-08 — Logging, Metriken & Fehler-Tracking.
//
// Dieses File enthält die drei Bausteine, die bisher gefehlt haben:
//
// 1. requestIDMiddleware: erzeugt (oder übernimmt aus X-Request-ID) eine
// Korrelations-ID, hängt sie an den context.Context und gibt sie im
// Response-Header zurück. Über loggerFromCtx(ctx) bekommt jede Log-Zeile
// im Request-Lebenszyklus das Feld request_id, ohne dass jede Call-Site
// umgeschrieben werden muss.
// 2. metricsMiddleware: zählt Requests je (Methode, normalisierter Pfad,
// Status) und summiert die Latenz in Histogramm-Buckets. Ausgabe über
// GET /metrics im Prometheus-Textformat (kein prometheus/client_golang).
// 3. recoverMiddleware: zentrales Panic-Recovery. net/http's ServeMux hat
// keins; ohne das reißt ein Panic in einem Handler die Verbindung ab und
// der Fehler taucht nirgends auf.
//
// Bewusst KEIN Logging von Query-Strings, Headern oder Request-Bodies:
// dort stecken Tokens (Signed-URL-Signatur, Share-Token, Bearer-Keys) und
// potenziell Passwörter. Geloggt werden ausschließlich Methode, normalisierter
// Pfad (IDs/Tokens durch Platzhalter ersetzt), Status, Dauer und Client-IP.
package api
import (
"context"
"crypto/rand"
"encoding/hex"
"log/slog"
"net/http"
"runtime/debug"
"strconv"
"strings"
"sync"
"time"
)
// stackTrace liefert einen gekürzten Stacktrace für das Panic-Log.
func stackTrace() string {
const maxStack = 4096
buf := debug.Stack()
if len(buf) > maxStack {
buf = buf[:maxStack]
}
return string(buf)
}
const (
requestIDKey contextKey = "request_id"
loggerKey contextKey = "logger"
requestIDHeader = "X-Request-ID"
maxRequestIDLen = 64
)
// newRequestID erzeugt eine zufällige 16-stellige Hex-ID. Fällt bei einem
// (praktisch unmöglichen) Fehler der Entropiequelle auf einen Zeitstempel
// zurück — eine Anfrage darf daran nie scheitern.
func newRequestID() string {
buf := make([]byte, 8)
if _, err := rand.Read(buf); err != nil {
return strconv.FormatInt(time.Now().UnixNano(), 36)
}
return hex.EncodeToString(buf)
}
// sanitizeRequestID übernimmt eine vom Client/Proxy gelieferte ID nur, wenn
// sie kurz und druckbar-alphanumerisch ist. Verhindert Log-Injection über
// Zeilenumbrüche im Header.
func sanitizeRequestID(v string) string {
v = strings.TrimSpace(v)
if v == "" || len(v) > maxRequestIDLen {
return ""
}
for _, r := range v {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
case r == '-' || r == '_' || r == '.':
default:
return ""
}
}
return v
}
// requestIDFromCtx liefert die Korrelations-ID der laufenden Anfrage oder "".
func requestIDFromCtx(ctx context.Context) string {
if ctx == nil {
return ""
}
v, _ := ctx.Value(requestIDKey).(string)
return v
}
// loggerFromCtx liefert den Request-Logger (inkl. request_id-Feld). Außerhalb
// eines HTTP-Requests — oder wenn kein Logger hinterlegt wurde — kommt ein
// no-op-freier Fallback zurück, damit Aufrufer nie auf nil prüfen müssen.
func loggerFromCtx(ctx context.Context) *slog.Logger {
if ctx != nil {
if l, ok := ctx.Value(loggerKey).(*slog.Logger); ok && l != nil {
return l
}
}
return slog.Default()
}
// reqLog ist der Einstieg für Handler: nutzt den Request-Logger aus dem
// Context, fällt aber auf den Server-Logger zurück, wenn die Middleware nicht
// durchlaufen wurde (z.B. in Tests, die Handler direkt aufrufen).
func (s *Server) reqLog(ctx context.Context) *slog.Logger {
if ctx != nil {
if l, ok := ctx.Value(loggerKey).(*slog.Logger); ok && l != nil {
return l
}
}
if s.logger != nil {
return s.logger
}
return slog.Default()
}
// requestIDMiddleware setzt die Korrelations-ID und den Request-Logger.
func (s *Server) requestIDMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rid := sanitizeRequestID(r.Header.Get(requestIDHeader))
if rid == "" {
rid = newRequestID()
}
base := s.logger
if base == nil {
base = slog.Default()
}
reqLogger := base.With("request_id", rid)
ctx := context.WithValue(r.Context(), requestIDKey, rid)
ctx = context.WithValue(ctx, loggerKey, reqLogger)
w.Header().Set(requestIDHeader, rid)
next.ServeHTTP(w, r.WithContext(ctx))
})
}
// statusRecorder merkt sich Statuscode und geschriebene Bytes, damit die
// Metrik-Middleware nach dem Handler auswerten kann.
type statusRecorder struct {
http.ResponseWriter
status int
written int64
wrote bool
}
func (rec *statusRecorder) WriteHeader(code int) {
if !rec.wrote {
rec.status = code
rec.wrote = true
}
rec.ResponseWriter.WriteHeader(code)
}
func (rec *statusRecorder) Write(b []byte) (int, error) {
if !rec.wrote {
rec.status = http.StatusOK
rec.wrote = true
}
n, err := rec.ResponseWriter.Write(b)
rec.written += int64(n)
return n, err
}
// Flush reicht http.Flusher durch (Downloads/Streaming-Endpunkte).
func (rec *statusRecorder) Flush() {
if f, ok := rec.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
// recoverMiddleware fängt Panics zentral ab: Log mit Korrelations-ID +
// Stacktrace, sauberer 500 an den Client. Ohne das stürzt zwar nicht der
// Prozess (net/http fängt pro Verbindung ab), der Fehler bleibt aber
// unsichtbar und der Client bekommt einen abgebrochenen Stream.
func (s *Server) recoverMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
rec := recover()
if rec == nil {
return
}
if rec == http.ErrAbortHandler {
panic(rec)
}
s.metrics.incPanic()
loggerFromCtx(r.Context()).Error("panic in http handler",
"method", r.Method,
"path", normalizeRoute(r.URL.Path),
"remote_ip", s.remoteIP(r),
"panic", rec,
"stack", stackTrace(),
)
if sr, ok := w.(*statusRecorder); ok && sr.wrote {
return // Header sind raus, mehr geht nicht
}
writeError(w, http.StatusInternalServerError, "interner Serverfehler")
}()
next.ServeHTTP(w, r)
})
}
// metricsMiddleware misst Dauer und Status jeder Anfrage.
func (s *Server) metricsMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
start := time.Now()
s.metrics.incInFlight()
defer func() {
s.metrics.decInFlight()
d := time.Since(start)
route := normalizeRoute(r.URL.Path)
s.metrics.observe(r.Method, route, rec.status, d)
// Zugriffs-Log: JEDE Anfrage erzeugt genau eine Zeile mit
// request_id (AK1). Level nach Status gestaffelt, damit im
// Normalbetrieb (Info) nur Auffälligkeiten sichtbar sind:
// 5xx=Error, 4xx=Warn, Rest=Debug.
lvl := slog.LevelDebug
msg := "request completed"
switch {
case rec.status >= 500:
lvl, msg = slog.LevelError, "request failed"
case rec.status >= 400:
lvl, msg = slog.LevelWarn, "request rejected"
}
loggerFromCtx(r.Context()).Log(r.Context(), lvl, msg,
"method", r.Method, "route", route,
"status", rec.status, "duration_ms", d.Milliseconds(),
"bytes", rec.written,
"remote_ip", s.remoteIP(r))
}()
next.ServeHTTP(rec, r)
})
}
// --- Pfad-Normalisierung ---
// tokenSegments sind Pfadabschnitte, deren FOLGENDES Segment ein Geheimnis
// ist (Share-Token). Die dürfen niemals in Logs oder Metrik-Labels landen.
var tokenSegments = map[string]bool{"share": true}
// normalizeRoute ersetzt variable Pfadsegmente durch Platzhalter. Das hält
// die Label-Kardinalität der Metriken klein UND verhindert, dass IDs oder
// Share-Tokens in Logs/Metriken auftauchen. Query-Strings werden nie
// betrachtet (dort stehen Signaturen und API-Keys).
func normalizeRoute(path string) string {
if path == "" {
return "/"
}
parts := strings.Split(path, "/")
prevSecret := false
for i, p := range parts {
if p == "" {
continue
}
if prevSecret {
parts[i] = "{token}"
prevSecret = false
continue
}
prevSecret = tokenSegments[p]
if isVariableSegment(p) {
parts[i] = "{id}"
}
}
out := strings.Join(parts, "/")
if len(out) > 120 {
return "/other"
}
return out
}
// isVariableSegment erkennt IDs, Hashes und sonstige nicht-statische
// Pfadbestandteile. Konservativ: im Zweifel maskieren.
func isVariableSegment(seg string) bool {
if _, err := strconv.ParseInt(seg, 10, 64); err == nil {
return true
}
if len(seg) > 40 {
return true
}
hasDigit := false
for _, r := range seg {
switch {
case r >= '0' && r <= '9':
hasDigit = true
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r == '-', r == '_', r == '.':
default:
// Alles Ungewöhnliche (Sonderzeichen, Umlaute, %-Encoding) ist mit
// Sicherheit kein statisches Routensegment.
return true
}
}
// Statische Routensegmente sind reine Wörter ("delete-requests",
// "classification-templates"). Längere Mischungen aus Buchstaben und
// Ziffern sind Hashes/Tokens/Dateinamen -> maskieren.
return hasDigit && len(seg) >= 8
}
// --- Metrik-Registry ---
// latencyBuckets sind die oberen Grenzen (Sekunden) des Latenz-Histogramms.
var latencyBuckets = []float64{0.005, 0.025, 0.1, 0.5, 1, 2.5, 5, 10, 30}
type routeKey struct {
method string
route string
status int
}
type routeStat struct {
count uint64
sumSeconds float64
bucketCount []uint64 // len(latencyBuckets), kumulativ erst beim Rendern
}
// metricsRegistry ist eine minimale, prozesslokale Metrik-Sammlung. Keine
// globale Variable: hängt als Feld am Server (Dependency Injection).
type metricsRegistry struct {
mu sync.Mutex
routes map[routeKey]*routeStat
inFlight int64
panics uint64
started time.Time
}
func newMetricsRegistry() *metricsRegistry {
return &metricsRegistry{
routes: make(map[routeKey]*routeStat),
started: time.Now(),
}
}
func (m *metricsRegistry) observe(method, route string, status int, d time.Duration) {
if m == nil {
return
}
k := routeKey{method: method, route: route, status: status}
secs := d.Seconds()
m.mu.Lock()
defer m.mu.Unlock()
// Kardinalitätsbremse: unbekannte Pfade nicht unbegrenzt sammeln.
st := m.routes[k]
if st == nil {
if len(m.routes) >= 500 {
k = routeKey{method: method, route: "/other", status: status}
st = m.routes[k]
}
if st == nil {
st = &routeStat{bucketCount: make([]uint64, len(latencyBuckets))}
m.routes[k] = st
}
}
st.count++
st.sumSeconds += secs
for i, ub := range latencyBuckets {
if secs <= ub {
st.bucketCount[i]++
}
}
}
func (m *metricsRegistry) incInFlight() {
if m == nil {
return
}
m.mu.Lock()
m.inFlight++
m.mu.Unlock()
}
func (m *metricsRegistry) decInFlight() {
if m == nil {
return
}
m.mu.Lock()
m.inFlight--
m.mu.Unlock()
}
func (m *metricsRegistry) incPanic() {
if m == nil {
return
}
m.mu.Lock()
m.panics++
m.mu.Unlock()
}
// snapshot liefert eine Kopie der Zähler für das Rendern.
func (m *metricsRegistry) snapshot() (map[routeKey]routeStat, int64, uint64) {
m.mu.Lock()
defer m.mu.Unlock()
out := make(map[routeKey]routeStat, len(m.routes))
for k, v := range m.routes {
cp := routeStat{count: v.count, sumSeconds: v.sumSeconds, bucketCount: make([]uint64, len(v.bucketCount))}
copy(cp.bucketCount, v.bucketCount)
out[k] = cp
}
return out, m.inFlight, m.panics
}
+174
View File
@@ -0,0 +1,174 @@
// FDN-08 — Tests zu den drei Akzeptanzkriterien:
// 1. Korrelations-ID über alle Schichten
// 2. Metriken (Latenz, Fehlerrate, Queue-Länge)
// 3. Unbehandelte Fehler werden zentral gemeldet
//
// Zusätzlich Prüfung 2 der Abnahme: es dürfen keine Tokens/Passwörter in Logs
// oder Metrik-Labels landen (TestNormalizeRouteRedactsSecrets).
package api
import (
"bytes"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"testing"
"archivdms/config"
)
func newTestServer(buf *bytes.Buffer) *Server {
logger := slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
return New(config.APIConfig{}, nil, nil, nil, nil, logger)
}
// AK 1: jede Anfrage bekommt eine Korrelations-ID, eine vom Client gelieferte
// wird übernommen und im Response-Header zurückgegeben.
func TestRequestIDGeneratedAndEchoed(t *testing.T) {
srv := newTestServer(&bytes.Buffer{})
rec := httptest.NewRecorder()
srv.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/health", nil))
got := rec.Header().Get(requestIDHeader)
if got == "" {
t.Fatalf("erwartete generierte Request-ID im Header %s", requestIDHeader)
}
req := httptest.NewRequest(http.MethodGet, "/api/health", nil)
req.Header.Set(requestIDHeader, "abc-123")
rec2 := httptest.NewRecorder()
srv.ServeHTTP(rec2, req)
if rec2.Header().Get(requestIDHeader) != "abc-123" {
t.Fatalf("Client-Request-ID nicht übernommen: %q", rec2.Header().Get(requestIDHeader))
}
// Log-Injection: unsaubere IDs werden verworfen, nicht durchgereicht.
bad := httptest.NewRequest(http.MethodGet, "/api/health", nil)
bad.Header.Set(requestIDHeader, "evil\nid")
rec3 := httptest.NewRecorder()
srv.ServeHTTP(rec3, bad)
if strings.Contains(rec3.Header().Get(requestIDHeader), "evil") {
t.Fatalf("ungültige Request-ID wurde übernommen: %q", rec3.Header().Get(requestIDHeader))
}
}
// AK 1: die ID landet im Logger, den Handler über den Context ziehen.
func TestLoggerFromCtxCarriesRequestID(t *testing.T) {
var buf bytes.Buffer
srv := newTestServer(&buf)
h := srv.requestIDMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if requestIDFromCtx(r.Context()) == "" {
t.Errorf("keine Request-ID im Context")
}
srv.reqLog(r.Context()).Info("testereignis")
}))
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/health", nil))
rid := rec.Header().Get(requestIDHeader)
if !strings.Contains(buf.String(), "request_id="+rid) {
t.Fatalf("Log-Zeile ohne request_id=%s: %s", rid, buf.String())
}
}
// AK 2: Zähler und Latenz-Histogramm werden im Prometheus-Textformat geliefert.
func TestMetricsEndpoint(t *testing.T) {
srv := newTestServer(&bytes.Buffer{})
for i := 0; i < 2; i++ {
srv.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/api/health", nil))
}
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
req.RemoteAddr = "127.0.0.1:54321"
rec := httptest.NewRecorder()
srv.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("erwartet 200, bekam %d", rec.Code)
}
body := rec.Body.String()
want := `archivdms_http_requests_total{method="GET",route="/api/health",status="200"} 2`
if !strings.Contains(body, want) {
t.Fatalf("Request-Zähler fehlt:\n%s", body)
}
for _, frag := range []string{
"archivdms_http_request_duration_seconds_bucket",
"archivdms_http_request_duration_seconds_sum",
"archivdms_goroutines",
"archivdms_panics_total",
} {
if !strings.Contains(body, frag) {
t.Errorf("Metrik %q fehlt in der Ausgabe", frag)
}
}
}
// AK 2 (Zugriffsschutz): fremde IPs dürfen nicht scrapen.
func TestMetricsEndpointIPRestricted(t *testing.T) {
srv := newTestServer(&bytes.Buffer{})
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
req.RemoteAddr = "203.0.113.7:5000"
rec := httptest.NewRecorder()
srv.ServeHTTP(rec, req)
if rec.Code != http.StatusForbidden {
t.Fatalf("erwartet 403 für fremde IP, bekam %d", rec.Code)
}
srv.cfg.MetricsAllowedIPs = []string{"203.0.113.0/24"}
rec2 := httptest.NewRecorder()
req2 := httptest.NewRequest(http.MethodGet, "/metrics", nil)
req2.RemoteAddr = "203.0.113.7:5000"
srv.ServeHTTP(rec2, req2)
if rec2.Code != http.StatusOK {
t.Fatalf("erwartet 200 für freigeschaltete CIDR, bekam %d", rec2.Code)
}
}
// AK 3: ein Panic wird zentral abgefangen, mit Korrelations-ID geloggt und als
// sauberer 500 beantwortet — der Zähler archivdms_panics_total steigt.
func TestRecoverMiddlewareCatchesPanic(t *testing.T) {
var buf bytes.Buffer
srv := newTestServer(&buf)
boom := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
panic("kaputt")
})
h := srv.requestIDMiddleware(srv.metricsMiddleware(srv.recoverMiddleware(boom)))
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/documents/42", nil))
if rec.Code != http.StatusInternalServerError {
t.Fatalf("erwartet 500, bekam %d", rec.Code)
}
rid := rec.Header().Get(requestIDHeader)
logged := buf.String()
if !strings.Contains(logged, "panic in http handler") || !strings.Contains(logged, "request_id="+rid) {
t.Fatalf("Panic nicht mit Korrelations-ID geloggt: %s", logged)
}
if _, _, panics := srv.metrics.snapshot(); panics != 1 {
t.Fatalf("erwartet 1 gezähltes Panic, bekam %d", panics)
}
}
// Abnahme-Prüfung 2: keine Tokens/IDs in Logs oder Metrik-Labels.
func TestNormalizeRouteRedactsSecrets(t *testing.T) {
cases := [][2]string{
{"/api/health", "/api/health"},
{"/api/documents/42", "/api/documents/{id}"},
{"/api/documents/42/notes/7", "/api/documents/{id}/notes/{id}"},
{"/api/classification-templates", "/api/classification-templates"},
{"/api/trash/9/delete-requests", "/api/trash/{id}/delete-requests"},
{"/public/share/s3cr3tTokenXyz", "/public/share/{token}"},
{"/public/share/abc/download", "/public/share/{token}/download"},
}
for _, c := range cases {
in, want := c[0], c[1]
if got := normalizeRoute(in); got != want {
t.Errorf("normalizeRoute(%q) = %q, erwartet %q", in, got, want)
}
}
}
+1 -1
View File
@@ -51,7 +51,7 @@ func (s *Server) handleListDocumentOCRWords(w http.ResponseWriter, r *http.Reque
words, err := s.store.ListOCRWords(r.Context(), id)
if err != nil {
s.logger.Error("list ocr words failed", "document_id", id, "tenant_id", *sess.TenantID, "err", err)
s.reqLog(r.Context()).Error("list ocr words failed", "document_id", id, "tenant_id", *sess.TenantID, "err", err)
writeError(w, http.StatusInternalServerError, "list ocr words failed")
return
}
+5 -6
View File
@@ -15,7 +15,6 @@ import (
"errors"
"io"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
@@ -107,7 +106,7 @@ func (s *Server) handlePublicShareDownload(w http.ResponseWriter, r *http.Reques
// Atomically claim one access slot (closes the max_accesses race).
ok, err := s.store.IncrementShareAccess(r.Context(), rs.ShareID())
if err != nil {
s.logger.Error("share access increment failed", "share_id", rs.ShareID(), "err", err)
s.reqLog(r.Context()).Error("share access increment failed", "share_id", rs.ShareID(), "err", err)
writeError(w, http.StatusInternalServerError, "download failed")
return
}
@@ -118,9 +117,9 @@ func (s *Server) handlePublicShareDownload(w http.ResponseWriter, r *http.Reques
return
}
f, err := os.Open(rs.StoragePath())
f, err := s.objects.Open(r.Context(), rs.TenantID(), rs.StoragePath())
if err != nil {
s.logger.Error("share file open failed", "share_id", rs.ShareID(), "err", err)
s.reqLog(r.Context()).Error("share file open failed", "share_id", rs.ShareID(), "err", err)
writeError(w, http.StatusInternalServerError, "download failed")
return
}
@@ -133,7 +132,7 @@ func (s *Server) handlePublicShareDownload(w http.ResponseWriter, r *http.Reques
w.Header().Set("Content-Disposition", "attachment; filename=\""+safeDownloadName(rs.DocumentTitle, ext)+"\"")
w.Header().Set("X-Content-Type-Options", "nosniff")
if _, err := io.Copy(w, f); err != nil {
s.logger.Warn("share file stream interrupted", "share_id", rs.ShareID(), "err", err)
s.reqLog(r.Context()).Warn("share file stream interrupted", "share_id", rs.ShareID(), "err", err)
}
}
@@ -141,7 +140,7 @@ func (s *Server) handlePublicShareDownload(w http.ResponseWriter, r *http.Reques
// into the audit log (EventShareAccessed). Never blocks the response path.
func (s *Server) recordShareAccess(r *http.Request, rs *storage.ResolvedShare, ip, result string) {
if err := s.store.LogShareAccess(r.Context(), rs.ShareID(), ip, r.UserAgent(), result); err != nil {
s.logger.Error("share access log failed", "share_id", rs.ShareID(), "err", err)
s.reqLog(r.Context()).Error("share access log failed", "share_id", rs.ShareID(), "err", err)
}
tenantID := rs.TenantID()
s.audlog.Log(audit.Entry{
+83 -1
View File
@@ -6,10 +6,13 @@ package api
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"log/slog"
"net"
"net/http"
"strconv"
"strings"
"time"
@@ -19,6 +22,7 @@ import (
"archivdms/internal/ldapauth"
"archivdms/internal/ldapstore"
"archivdms/internal/mailer"
"archivdms/internal/objectstore"
"archivdms/internal/ocr"
"archivdms/internal/pagesplit"
"archivdms/internal/storage"
@@ -46,6 +50,10 @@ type Server struct {
logger *slog.Logger
mux *http.ServeMux
ocr *ocr.Extractor
// objects is the WORM object-storage driver (internal/objectstore): the
// only place archived files are written, read or unlinked. Wired by
// SetStorageConfig/SetObjectStore.
objects objectstore.Store
thumbs *thumbnail.Generator
// pagesplitter performs barcode separator-page splitting of multi-page PDF
// uploads before archival (internal/pagesplit). May be nil / disabled, in
@@ -66,6 +74,15 @@ type Server struct {
// (per client IP) to blunt token/password enumeration.
shareLimiter *ipRateLimiter
// metrics ist die prozesslokale Metrik-Registry (FDN-08,
// internal/api/observability.go). Kein globaler Zustand: hängt am Server.
metrics *metricsRegistry
// baseHandler ist die in New() gebaute Middleware-Kette um s.mux
// (requestID -> metrics -> recover). Einmal gebaut, danach nur gelesen —
// kein Lazy-Init in ServeHTTP (Data Race).
baseHandler http.Handler
// accountingLimiter rate-limits the Bearer-key Buchhaltungs-Pull-API
// (per client IP) to blunt API-key guessing. Separate bucket set from
// shareLimiter so a busy accounting client cannot starve share downloads.
@@ -76,8 +93,46 @@ type Server struct {
// paths, max upload size) into the API server. Needed by
// handleUploadDocument, which cannot rely solely on the storage.Store
// (that only knows its own base dir, not the inbox/ocr-tmp layout).
//
// It also constructs the object-storage driver (internal/objectstore), the
// single place where archived files are written, read and unlinked. Call
// SetFQDN before this if generated signed URLs should be absolute.
func (s *Server) SetStorageConfig(cfg config.StorageConfig) {
s.storageCfg = cfg
secret := s.cfg.Secret
if strings.TrimSpace(secret) == "" {
// No master secret configured: fall back to an ephemeral, per-process
// key so file access keeps working; signed URLs then simply do not
// survive a restart. Never fail startup over this.
buf := make([]byte, 32)
if _, err := rand.Read(buf); err != nil {
secret = strconv.FormatInt(time.Now().UnixNano(), 36)
} else {
secret = hex.EncodeToString(buf)
}
s.logger.Warn("objectstore: api.secret unset, using ephemeral signing key")
}
driver, err := objectstore.NewLocalStore(cfg, secret, s.publicBaseURL())
if err != nil {
s.logger.Error("objectstore init failed", "err", err)
return
}
s.objects = driver
}
// SetObjectStore overrides the object-storage driver (tests / alternative
// wiring). Normally set implicitly by SetStorageConfig.
func (s *Server) SetObjectStore(o objectstore.Store) {
s.objects = o
}
// publicBaseURL is the origin used for generated links. Empty FQDN yields
// site-relative URLs.
func (s *Server) publicBaseURL() string {
if strings.TrimSpace(s.fqdn) == "" {
return ""
}
return "https://" + strings.TrimSuffix(strings.TrimSpace(s.fqdn), "/")
}
// SetOCR wires the OCR extractor into the API server. May be nil, in which
@@ -150,8 +205,10 @@ func New(
shareLimiter: newIPRateLimiter(20, 1.0),
// Batch pulls are legitimate here: 60 requests burst, refilled at 5/sec.
accountingLimiter: newIPRateLimiter(60, 5.0),
metrics: newMetricsRegistry(),
}
s.routes()
s.baseHandler = s.requestIDMiddleware(s.metricsMiddleware(s.recoverMiddleware(s.mux)))
return s
}
@@ -168,6 +225,10 @@ func (s *Server) authAdmin(h http.HandlerFunc) http.HandlerFunc {
func (s *Server) routes() {
s.mux.HandleFunc("GET /api/health", s.handleHealth)
s.mux.HandleFunc("GET /api/version", s.handleVersion)
// Prometheus-Scrape-Endpunkt (FDN-08, internal/api/metrics_handlers.go).
// Bewusst ohne s.auth — ein Scraper hat keine Session; der Zugriff wird
// stattdessen per Quell-IP begrenzt (loopback + api.metrics_allowed_ips).
s.mux.HandleFunc("GET /metrics", s.handleMetrics)
s.mux.HandleFunc("POST /api/auth/login", s.handleLogin)
s.mux.HandleFunc("GET /api/auth/me", s.auth(s.handleMe))
@@ -377,6 +438,12 @@ func (s *Server) routes() {
s.mux.HandleFunc("GET /public/share/{token}", s.handlePublicShareMeta)
s.mux.HandleFunc("POST /public/share/{token}/download", s.handlePublicShareDownload)
// Signed, time-limited download URLs (internal/api/signed_url_handlers.go).
// Issuing is authenticated + tenant-scoped; redeeming runs WITHOUT s.auth
// because the HMAC signature in the query string is the credential.
s.mux.HandleFunc("POST /api/documents/{id}/signed-url", s.auth(s.handleCreateDocumentSignedURL))
s.mux.HandleFunc("GET /public/files", s.handleSignedFileDownload)
// Buchhaltungs-Pull-API (internal/api/accounting_handlers.go).
// Key administration runs on the normal session auth and is domain_admin-only
// (a key grants tenant-wide read access to archived documents).
@@ -417,8 +484,23 @@ func (s *Server) routes() {
}
// ServeHTTP implements http.Handler.
//
// Die Basis-Middleware-Kette (FDN-08) liegt bewusst hier und nicht an den
// einzelnen Routen, damit sie ausnahmslos für JEDE Anfrage gilt — auch für
// /public/*, /metrics und nicht gefundene Pfade:
//
// requestID -> metrics -> recover -> ServeMux
//
// Reihenfolge: requestID zuerst, damit Metrik- und Panic-Log die
// Korrelations-ID haben; recover innen, damit der 500 noch über den
// statusRecorder der Metrik-Middleware läuft und dort gezählt wird.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.mux.ServeHTTP(w, r)
if s.baseHandler == nil {
// Server wurde nicht über New() gebaut (Tests): ohne Kette bedienen.
s.mux.ServeHTTP(w, r)
return
}
s.baseHandler.ServeHTTP(w, r)
}
// --- system handlers ---
+162
View File
@@ -0,0 +1,162 @@
// Signed, time-limited download URLs for archived documents (FDN-03,
// internal/objectstore). Same principle as the external share links
// (share_handlers.go / public_share_handlers.go) — unguessable credential in
// the link, mandatory hard expiry, IP rate limiting, audit trail — but
// stateless: the credential is an HMAC-SHA256 signature over
// tenant|document|expiry instead of a DB row.
//
// POST /api/documents/{id}/signed-url issue a link (authenticated, tenant-scoped)
// GET /public/files?t=&d=&exp=&sig= redeem it (no session, signature is the credential)
//
// Use share links when a document is handed to an external party with its own
// lifecycle (revoke, password, access cap); use signed URLs for short-lived
// machine access to the file itself (viewer, export job) without a session
// cookie. Nothing here bypasses tenant scoping: the tenant id is taken from
// the signed payload and every lookup still filters on it.
package api
import (
"encoding/json"
"errors"
"io"
"net/http"
"path/filepath"
"strconv"
"time"
"archivdms/internal/audit"
"archivdms/internal/objectstore"
)
// createSignedURLRequest is the optional POST body. TTLMinutes overrides the
// configured default validity (storage.signed_url_ttl_minutes); it is capped
// at 24 hours so no effectively unbounded link can be minted.
type createSignedURLRequest struct {
TTLMinutes int `json:"ttl_minutes,omitempty"`
}
// createSignedURLResponse is returned to the caller.
type createSignedURLResponse struct {
URL string `json:"url"`
ExpiresAt time.Time `json:"expires_at"`
}
// maxSignedURLTTL caps a caller-supplied validity.
const maxSignedURLTTL = 24 * time.Hour
// handleCreateDocumentSignedURL handles POST /api/documents/{id}/signed-url.
func (s *Server) handleCreateDocumentSignedURL(w http.ResponseWriter, r *http.Request) {
sess := sessionFromCtx(r.Context())
if sess.TenantID == nil {
writeError(w, http.StatusForbidden, "tenant context required")
return
}
if s.objects == nil {
writeError(w, http.StatusServiceUnavailable, "Dateispeicher nicht konfiguriert")
return
}
docID, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid document id")
return
}
var req createSignedURLRequest
if r.Body != nil {
_ = json.NewDecoder(r.Body).Decode(&req) // body is optional
}
ttl := time.Duration(req.TTLMinutes) * time.Minute
if ttl <= 0 {
ttl = s.storageCfg.ResolvedSignedURLTTL()
}
if ttl > maxSignedURLTTL {
ttl = maxSignedURLTTL
}
// IDOR guard: the document must belong to the caller's tenant. GetDocument
// filters WHERE id = $1 AND tenant_id = $2.
doc, err := s.store.GetDocument(r.Context(), docID, *sess.TenantID)
if err != nil {
s.logShare(r, audit.EventSignedURLCreated, sess.TenantID, sess.Username,
"signed_url doc:"+strconv.FormatInt(docID, 10)+" err:not_found", false)
writeError(w, http.StatusNotFound, "document not found")
return
}
link, err := s.objects.SignedURL(*sess.TenantID, doc.ID, ttl)
if err != nil {
s.logShare(r, audit.EventSignedURLCreated, sess.TenantID, sess.Username,
"signed_url doc:"+strconv.FormatInt(docID, 10)+" err:"+err.Error(), false)
writeError(w, http.StatusInternalServerError, "Link konnte nicht erstellt werden")
return
}
expiresAt := time.Now().Add(ttl)
s.logShare(r, audit.EventSignedURLCreated, sess.TenantID, sess.Username,
"signed_url doc:"+strconv.FormatInt(docID, 10)+" ttl:"+ttl.String(), true)
writeJSON(w, http.StatusOK, createSignedURLResponse{URL: link, ExpiresAt: expiresAt})
}
// handleSignedFileDownload handles GET /public/files?t=&d=&exp=&sig=. Served
// WITHOUT the s.auth wrapper by design: the signature is the credential.
// Order: rate-limit -> verify signature -> verify expiry -> tenant-scoped
// document lookup -> stream from the WORM store. Every outcome is audit-logged
// (EventSignedURLAccessed), successes and failures alike.
func (s *Server) handleSignedFileDownload(w http.ResponseWriter, r *http.Request) {
ip := s.remoteIP(r)
if !s.shareLimiter.allow(ip) {
writeError(w, http.StatusTooManyRequests, "too many requests")
return
}
if s.objects == nil {
writeError(w, http.StatusServiceUnavailable, "Dateispeicher nicht konfiguriert")
return
}
ref, err := s.objects.VerifySignedURL(r.URL.Query(), time.Now())
if err != nil {
s.logSignedAccess(r, nil, "signed_url_access err:"+err.Error(), false)
if errors.Is(err, objectstore.ErrSignatureExpired) {
writeError(w, http.StatusGone, "Der Link ist abgelaufen.")
return
}
writeError(w, http.StatusForbidden, "Der Link ist ungültig.")
return
}
doc, err := s.store.GetDocument(r.Context(), ref.DocumentID, ref.TenantID)
if err != nil {
s.logSignedAccess(r, &ref.TenantID,
"signed_url_access doc:"+strconv.FormatInt(ref.DocumentID, 10)+" err:not_found", false)
writeError(w, http.StatusNotFound, "Dokument nicht gefunden.")
return
}
f, err := s.objects.Open(r.Context(), ref.TenantID, doc.StoragePath)
if err != nil {
s.reqLog(r.Context()).Error("signed url file open failed", "document_id", doc.ID, "tenant_id", ref.TenantID, "err", err)
s.logSignedAccess(r, &ref.TenantID,
"signed_url_access doc:"+strconv.FormatInt(doc.ID, 10)+" err:"+err.Error(), false)
if errors.Is(err, objectstore.ErrObjectNotFound) {
writeError(w, http.StatusNotFound, "Datei nicht im Archiv vorhanden.")
return
}
writeError(w, http.StatusInternalServerError, "Download fehlgeschlagen.")
return
}
defer f.Close()
s.logSignedAccess(r, &ref.TenantID, "signed_url_access doc:"+strconv.FormatInt(doc.ID, 10), true)
ext := filepath.Ext(doc.StoragePath)
w.Header().Set("Content-Type", detectMimeType("", ext, doc.StoragePath))
w.Header().Set("Content-Disposition", "attachment; filename=\""+safeDownloadName(doc.Title, ext)+"\"")
w.Header().Set("X-Content-Type-Options", "nosniff")
if _, err := io.Copy(w, f); err != nil {
s.reqLog(r.Context()).Warn("signed url stream interrupted", "document_id", doc.ID, "err", err)
}
}
// logSignedAccess records a redemption attempt in the audit log. There is no
// session here, so the username column carries the anonymous marker.
func (s *Server) logSignedAccess(r *http.Request, tenantID *int64, detail string, ok bool) {
s.audlog.Log(audit.Entry{
EventType: audit.EventSignedURLAccessed, Username: "anonymous", TenantID: tenantID,
IPAddress: s.remoteIP(r), Success: ok, Detail: detail,
})
}