FDN-02/FDN-03/FDN-07/FDN-08: Migrations-Rollback, Objekt-Storage-Interface, go.sum-Fix, Observability
- FDN-02: Rollback-fähige Down-Migrationen (024-026), archivdms seed dev CLI - FDN-03: internal/objectstore Interface + lokaler WORM-Treiber, signierte Download-URLs - FDN-07: go.mod/go.sum vervollständigt (fehlender go-ldap/v3-Eintrag), CI-Pipeline (.gitea/workflows/ci.yml, bereits in FDN-01 committet) damit lauffähig - FDN-08: Request-ID-Middleware, /metrics-Endpoint, Panic-Recovery, Login/Logout/Me technisches Logging inkl. Access-Log je Anfrage
This commit is contained in:
@@ -190,7 +190,7 @@ func (s *Server) accountingAuth(h http.HandlerFunc) http.HandlerFunc {
|
||||
tenantID, keyID, err := s.store.ResolveAccountingAPIKey(r.Context(), rawKey)
|
||||
if err != nil {
|
||||
if !errors.Is(err, storage.ErrAccountingKeyNotFound) {
|
||||
s.logger.Error("accounting api key resolve failed", "err", err)
|
||||
s.reqLog(r.Context()).Error("accounting api key resolve failed", "err", err)
|
||||
}
|
||||
// Unknown, revoked and broken keys are indistinguishable.
|
||||
s.audlog.Log(audit.Entry{
|
||||
@@ -290,7 +290,7 @@ func (s *Server) handleAccountingListDocuments(w http.ResponseWriter, r *http.Re
|
||||
writeError(w, http.StatusBadRequest, "invalid cursor")
|
||||
return
|
||||
}
|
||||
s.logger.Error("accounting list failed", "tenant_id", tenantID, "err", err)
|
||||
s.reqLog(r.Context()).Error("accounting list failed", "tenant_id", tenantID, "err", err)
|
||||
s.audlog.Log(audit.Entry{
|
||||
EventType: audit.EventAccountingPull, Username: "accounting_api", IPAddress: s.remoteIP(r),
|
||||
TenantID: &tenantID, Success: false,
|
||||
@@ -342,7 +342,7 @@ func (s *Server) handleAccountingDocumentFile(w http.ResponseWriter, r *http.Req
|
||||
|
||||
f, err := os.Open(ref.StoragePath())
|
||||
if err != nil {
|
||||
s.logger.Error("accounting file open failed", "document_id", ref.DocumentID, "tenant_id", tenantID, "err", err)
|
||||
s.reqLog(r.Context()).Error("accounting file open failed", "document_id", ref.DocumentID, "tenant_id", tenantID, "err", err)
|
||||
s.audlog.Log(audit.Entry{
|
||||
EventType: audit.EventAccountingPull, Username: "accounting_api", IPAddress: s.remoteIP(r),
|
||||
TenantID: &tenantID, DocumentID: strconv.FormatInt(id, 10), Success: false,
|
||||
@@ -364,7 +364,7 @@ func (s *Server) handleAccountingDocumentFile(w http.ResponseWriter, r *http.Req
|
||||
w.Header().Set("Content-Disposition", "attachment; filename=\""+safeDownloadName(ref.Title, ext)+"\"")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
if _, err := io.Copy(w, f); err != nil {
|
||||
s.logger.Warn("accounting file stream interrupted", "document_id", ref.DocumentID, "err", err)
|
||||
s.reqLog(r.Context()).Warn("accounting file stream interrupted", "document_id", ref.DocumentID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,15 +13,22 @@ type loginRequest struct {
|
||||
}
|
||||
|
||||
func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
var req loginRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
s.reqLog(ctx).Warn("login: invalid request body",
|
||||
"remote_ip", s.remoteIP(r), "err", err)
|
||||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
ip := s.remoteIP(r)
|
||||
token, user, err := s.authMgr.LoginFrom(r.Context(), req.Username, req.Password, ip)
|
||||
token, user, err := s.authMgr.LoginFrom(ctx, req.Username, req.Password, ip)
|
||||
if err != nil {
|
||||
// Kein Passwort, keine Fehlerdetails vom Auth-Manager im Klartext:
|
||||
// nur Benutzername + IP zur Korrelation von Brute-Force-Versuchen.
|
||||
s.reqLog(ctx).Warn("login failed",
|
||||
"username", req.Username, "remote_ip", ip, "reason", "invalid_credentials")
|
||||
s.audlog.Log(audit.Entry{
|
||||
EventType: audit.EventLogin,
|
||||
Username: req.Username,
|
||||
@@ -43,7 +50,14 @@ func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
|
||||
MaxAge: 8 * 60 * 60,
|
||||
})
|
||||
|
||||
_ = s.users.UpdateLastLogin(user.ID)
|
||||
if err := s.users.UpdateLastLogin(user.ID); err != nil {
|
||||
s.reqLog(ctx).Warn("login: last_login update failed",
|
||||
"user_id", user.ID, "err", err)
|
||||
}
|
||||
|
||||
s.reqLog(ctx).Info("login succeeded",
|
||||
"user_id", user.ID, "username", user.Username,
|
||||
"tenant_id", user.TenantID, "remote_ip", ip)
|
||||
|
||||
s.audlog.Log(audit.Entry{
|
||||
EventType: audit.EventLogin,
|
||||
@@ -57,9 +71,12 @@ func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleMe(w http.ResponseWriter, r *http.Request) {
|
||||
sess := sessionFromCtx(r.Context())
|
||||
ctx := r.Context()
|
||||
sess := sessionFromCtx(ctx)
|
||||
user, err := s.users.GetByID(sess.UserID)
|
||||
if err != nil {
|
||||
s.reqLog(ctx).Warn("me: user lookup failed",
|
||||
"user_id", sess.UserID, "err", err)
|
||||
writeError(w, http.StatusNotFound, "user not found")
|
||||
return
|
||||
}
|
||||
@@ -67,7 +84,8 @@ func (s *Server) handleMe(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request) {
|
||||
sess := sessionFromCtx(r.Context())
|
||||
ctx := r.Context()
|
||||
sess := sessionFromCtx(ctx)
|
||||
token := ""
|
||||
if c, err := r.Cookie(sessionCookieName); err == nil {
|
||||
token = c.Value
|
||||
@@ -76,9 +94,16 @@ func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request) {
|
||||
token = extractBearerToken(r)
|
||||
}
|
||||
if token != "" {
|
||||
_ = s.authMgr.Logout(token)
|
||||
if err := s.authMgr.Logout(token); err != nil {
|
||||
s.reqLog(ctx).Warn("logout: session invalidation failed",
|
||||
"user_id", sess.UserID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
s.reqLog(ctx).Info("logout",
|
||||
"user_id", sess.UserID, "username", sess.Username,
|
||||
"tenant_id", sess.TenantID, "remote_ip", s.remoteIP(r))
|
||||
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: sessionCookieName,
|
||||
Value: "",
|
||||
|
||||
@@ -21,7 +21,7 @@ func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
stats, err := s.store.GetDashboardStats(r.Context(), *sess.TenantID, sess.UserID)
|
||||
if err != nil {
|
||||
s.logger.Error("dashboard stats failed", "err", err)
|
||||
s.reqLog(r.Context()).Error("dashboard stats failed", "err", err)
|
||||
writeError(w, http.StatusInternalServerError, "dashboard stats failed")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -197,7 +197,7 @@ func (s *Server) handleBulkExportDocuments(w http.ResponseWriter, r *http.Reques
|
||||
if err != nil {
|
||||
// A single unreadable document must not kill the archive — unless the
|
||||
// ZIP writer itself failed, which we detect on Close below.
|
||||
s.logger.Warn("bulk export: document skipped", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
s.reqLog(r.Context()).Warn("bulk export: document skipped", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
skipped = append(skipped, fmt.Sprintf("%d: %v", doc.ID, err))
|
||||
continue
|
||||
}
|
||||
@@ -225,7 +225,7 @@ func (s *Server) handleBulkExportDocuments(w http.ResponseWriter, r *http.Reques
|
||||
detail := fmt.Sprintf("zip_bulk_export: exported=%d skipped=%d", exported, len(skipped))
|
||||
if streamErr != nil {
|
||||
// Headers are already out — audit the partial export, no HTTP error.
|
||||
s.logger.Warn("bulk document export stream failed", "tenant_id", tenantID, "err", streamErr)
|
||||
s.reqLog(r.Context()).Warn("bulk document export stream failed", "tenant_id", tenantID, "err", streamErr)
|
||||
s.audlog.Log(audit.Entry{
|
||||
EventType: audit.EventDocumentBulkExport, Username: sess.Username, TenantID: sess.TenantID,
|
||||
Success: false, Detail: detail + " stream_failed: " + streamErr.Error(),
|
||||
|
||||
@@ -143,7 +143,7 @@ func (s *Server) handleExportDocument(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
f, err := os.Open(doc.StoragePath)
|
||||
if err != nil {
|
||||
s.logger.Error("export: document file open failed", "document_id", doc.ID, "tenant_id", *sess.TenantID, "err", err)
|
||||
s.reqLog(r.Context()).Error("export: document file open failed", "document_id", doc.ID, "tenant_id", *sess.TenantID, "err", err)
|
||||
fail(http.StatusInternalServerError, "file unavailable", "file_open_failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
@@ -212,7 +212,7 @@ func (s *Server) handleExportDocument(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
if streamErr != nil {
|
||||
// Headers are already out — log + audit the partial export, no HTTP error.
|
||||
s.logger.Warn("document export stream failed", "document_id", doc.ID, "err", streamErr)
|
||||
s.reqLog(r.Context()).Warn("document export stream failed", "document_id", doc.ID, "err", streamErr)
|
||||
s.audlog.Log(audit.Entry{
|
||||
EventType: audit.EventDocumentExport, Username: sess.Username, TenantID: sess.TenantID,
|
||||
DocumentID: idStr, Success: false, Detail: "stream_failed: " + streamErr.Error(),
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"archivdms/internal/auth"
|
||||
"archivdms/internal/dateformat"
|
||||
"archivdms/internal/matching"
|
||||
"archivdms/internal/objectstore"
|
||||
"archivdms/internal/ocr"
|
||||
"archivdms/internal/storage"
|
||||
"archivdms/internal/userstore"
|
||||
@@ -138,10 +139,12 @@ func (s *Server) handleGetDocumentFile(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
f, err := os.Open(doc.StoragePath)
|
||||
// Read through the storage abstraction: it re-checks that the stored path
|
||||
// really lies inside this tenant's store subtree before opening.
|
||||
f, err := s.objects.Open(r.Context(), *sess.TenantID, doc.StoragePath)
|
||||
if err != nil {
|
||||
// WORM store should always hold the file, but never trust the disk.
|
||||
s.logger.Error("document file open failed", "document_id", doc.ID, "tenant_id", *sess.TenantID, "storage_path", doc.StoragePath, "err", err)
|
||||
s.reqLog(r.Context()).Error("document file open failed", "document_id", doc.ID, "tenant_id", *sess.TenantID, "storage_path", doc.StoragePath, "err", err)
|
||||
writeError(w, http.StatusInternalServerError, "file unavailable")
|
||||
return
|
||||
}
|
||||
@@ -152,7 +155,7 @@ func (s *Server) handleGetDocumentFile(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Disposition", "inline; filename=\""+safeDownloadName(doc.Title, ext)+"\"")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
if _, err := io.Copy(w, f); err != nil {
|
||||
s.logger.Warn("document file stream interrupted", "document_id", doc.ID, "err", err)
|
||||
s.reqLog(r.Context()).Warn("document file stream interrupted", "document_id", doc.ID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,7 +212,7 @@ func (s *Server) handleGetDocumentThumbnail(w http.ResponseWriter, r *http.Reque
|
||||
w.Header().Set("Cache-Control", "private, max-age=86400")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
if _, err := io.Copy(w, f); err != nil {
|
||||
s.logger.Warn("thumbnail stream interrupted", "document_id", doc.ID, "err", err)
|
||||
s.reqLog(r.Context()).Warn("thumbnail stream interrupted", "document_id", doc.ID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -453,7 +456,7 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
|
||||
mimeType := detectMimeType("", ext, doc.StoragePath)
|
||||
result, err := s.ocr.Extract(ctx, doc.StoragePath, mimeType)
|
||||
if err != nil {
|
||||
s.logger.Warn("reprocess ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID, "storage_path", doc.StoragePath, "err", err)
|
||||
s.reqLog(ctx).Warn("reprocess ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID, "storage_path", doc.StoragePath, "err", err)
|
||||
s.audlog.Log(audit.Entry{EventType: audit.EventDocumentReprocessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "ocr_failed: " + err.Error()})
|
||||
return nil, fmt.Errorf("reprocess ocr extract: %w", err)
|
||||
}
|
||||
@@ -466,7 +469,7 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
|
||||
// alongside the new ones — best-effort, never fails the reprocess since
|
||||
// ocr_text is already the authoritative persisted result.
|
||||
if err := s.store.ReplaceOCRWords(ctx, id, ocrWordsFromResult(id, result.Words)); err != nil {
|
||||
s.logger.Warn("reprocess replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
s.reqLog(ctx).Warn("reprocess replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
}
|
||||
|
||||
if err := s.store.UpdateDocumentOCRText(ctx, id, tenantID, ocrText); err != nil {
|
||||
@@ -491,7 +494,7 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
|
||||
prefix, dateLayout := s.tenantScanTitleParams(ctx, tenantID)
|
||||
if newTitle := titleFromOCRText(ocrText, prefix, dateLayout); newTitle != "" && newTitle != doc.Title {
|
||||
if err := s.store.UpdateDocumentTitleAuto(ctx, id, tenantID, newTitle); err != nil {
|
||||
s.logger.Warn("reprocess title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
s.reqLog(ctx).Warn("reprocess title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
} else {
|
||||
doc.Title = newTitle
|
||||
}
|
||||
@@ -513,7 +516,7 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
|
||||
}
|
||||
if !sameDate(datePtr, doc.DocumentDate) {
|
||||
if err := s.store.UpdateDocumentDate(ctx, id, tenantID, datePtr, scorePtr); err != nil {
|
||||
s.logger.Warn("reprocess document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
s.reqLog(ctx).Warn("reprocess document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
} else {
|
||||
doc.DocumentDate = datePtr
|
||||
doc.DocumentDateScore = scorePtr
|
||||
@@ -524,12 +527,12 @@ func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, acto
|
||||
// Additive auto-assignment (only fills unset doc_type/correspondent, only
|
||||
// attaches tags) — best-effort, never fails the request.
|
||||
if assignWarn := s.autoAssignTaxonomy(ctx, tenantID, doc, barcodes, ocrText); assignWarn != "" {
|
||||
s.logger.Info("reprocess auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn)
|
||||
s.reqLog(ctx).Info("reprocess auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn)
|
||||
}
|
||||
|
||||
// Re-evaluate on_upload workflows — best-effort, never fails the request.
|
||||
if err := s.store.RunWorkflowsForDocument(ctx, tenantID, doc, storage.WorkflowTriggerOnUpload); err != nil {
|
||||
s.logger.Warn("reprocess workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
s.reqLog(ctx).Warn("reprocess workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
}
|
||||
|
||||
// Re-read so the response reflects any doc_type/correspondent set by
|
||||
@@ -726,11 +729,11 @@ func (s *Server) generateThumbnailBestEffort(ctx context.Context, tenantID, docu
|
||||
}
|
||||
thumbPath := filepath.Join(s.storageCfg.ThumbnailPath(), strconv.FormatInt(tenantID, 10), contentHash+".png")
|
||||
if err := s.thumbs.Generate(ctx, storagePath, mimeType, thumbPath); err != nil {
|
||||
s.logger.Warn("eager thumbnail generation failed", "document_id", documentID, "tenant_id", tenantID, "mime_type", mimeType, "err", err)
|
||||
s.reqLog(ctx).Warn("eager thumbnail generation failed", "document_id", documentID, "tenant_id", tenantID, "mime_type", mimeType, "err", err)
|
||||
return
|
||||
}
|
||||
if err := s.store.SetDocumentHasThumbnail(ctx, documentID, tenantID, true); err != nil {
|
||||
s.logger.Warn("has_thumbnail flag update failed", "document_id", documentID, "tenant_id", tenantID, "err", err)
|
||||
s.reqLog(ctx).Warn("has_thumbnail flag update failed", "document_id", documentID, "tenant_id", tenantID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -753,7 +756,7 @@ func (s *Server) handleUploadDocument(w http.ResponseWriter, r *http.Request) {
|
||||
tenantID := *sess.TenantID
|
||||
|
||||
maxBytes := int64(s.storageCfg.ResolvedMaxUploadSizeMB()) * 1024 * 1024
|
||||
s.logger.Info("upload request received", "username", sess.Username, "tenant_id", tenantID,
|
||||
s.reqLog(r.Context()).Info("upload request received", "username", sess.Username, "tenant_id", tenantID,
|
||||
"content_length", r.ContentLength, "max_bytes", maxBytes, "remote_addr", r.RemoteAddr)
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxBytes)
|
||||
@@ -763,7 +766,7 @@ func (s *Server) handleUploadDocument(w http.ResponseWriter, r *http.Request) {
|
||||
// hitting MaxBytesReader — both close the connection before any
|
||||
// later log/audit call would otherwise run, which previously left
|
||||
// zero trace of the failure in the backend logs (see DEVLOG 2026-07-15).
|
||||
s.logger.Warn("upload parse failed", "username", sess.Username, "tenant_id", tenantID,
|
||||
s.reqLog(r.Context()).Warn("upload parse failed", "username", sess.Username, "tenant_id", tenantID,
|
||||
"content_length", r.ContentLength, "max_bytes", maxBytes, "err", err)
|
||||
s.audlog.Log(audit.Entry{
|
||||
EventType: audit.EventDocumentCreate, Username: sess.Username, TenantID: sess.TenantID,
|
||||
@@ -956,41 +959,19 @@ func (s *Server) archiveStagedFile(
|
||||
}
|
||||
pathDate := time.Now()
|
||||
|
||||
// 4. Build the WORM target path store/<tenant>/<yyyy>/<mm>/<hash>.<ext>.
|
||||
storeDir := filepath.Join(s.storageCfg.StorePath(), strconv.FormatInt(tenantID, 10),
|
||||
fmt.Sprintf("%04d", pathDate.Year()), fmt.Sprintf("%02d", pathDate.Month()))
|
||||
if err := os.MkdirAll(storeDir, 0o750); err != nil {
|
||||
os.Remove(inboxPath)
|
||||
return nil, "", fmt.Errorf("create store dir: %w", err)
|
||||
}
|
||||
storePath := filepath.Join(storeDir, contentHash+ext)
|
||||
|
||||
// 5. Collision check: identical hash already stored -> reject as
|
||||
// duplicate before touching the DB (filesystem-level half of the
|
||||
// duplicate protection; the DB unique index is the other half).
|
||||
if _, err := os.Stat(storePath); err == nil {
|
||||
os.Remove(inboxPath)
|
||||
return nil, "", storage.ErrDuplicateContentHash
|
||||
} else if !os.IsNotExist(err) {
|
||||
os.Remove(inboxPath)
|
||||
return nil, "", fmt.Errorf("stat store path: %w", err)
|
||||
}
|
||||
|
||||
// 6. Move inbox -> store. Prefer atomic rename; fall back to copy+remove
|
||||
// if inbox/store ever end up on different filesystems/mounts.
|
||||
if err := os.Rename(inboxPath, storePath); err != nil {
|
||||
if copyErr := copyFile(inboxPath, storePath); copyErr != nil {
|
||||
os.Remove(inboxPath)
|
||||
return nil, "", fmt.Errorf("move file to store: rename failed (%v), copy fallback failed: %w", err, copyErr)
|
||||
// 4./5./6./7. WORM archival — delegated unchanged to the storage
|
||||
// abstraction (internal/objectstore): build the target path
|
||||
// store/<tenant>/<yyyy>/<mm>/<hash>.<ext>, reject an already-stored hash as
|
||||
// duplicate (filesystem half of the duplicate protection; the DB unique
|
||||
// index is the other half), move inbox -> store (rename, copy+remove
|
||||
// fallback across mounts) and finally chmod 0440. Path scheme and
|
||||
// semantics are identical to the previous inline implementation.
|
||||
storePath, err := s.objects.Archive(ctx, tenantID, inboxPath, ext, contentHash, pathDate)
|
||||
if err != nil {
|
||||
if errors.Is(err, objectstore.ErrObjectExists) {
|
||||
return nil, "", storage.ErrDuplicateContentHash
|
||||
}
|
||||
os.Remove(inboxPath)
|
||||
}
|
||||
|
||||
// 7. WORM lock: read-only, no write access for anyone once archived. This is
|
||||
// the ONLY chmod and it happens exactly once, after the file reaches its
|
||||
// final path — the file is never moved or renamed again afterwards.
|
||||
if err := os.Chmod(storePath, 0o440); err != nil {
|
||||
return nil, "", fmt.Errorf("chmod store file: %w", err)
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
// 8. Record the document AND its processing job in ONE transaction. If no
|
||||
@@ -1018,7 +999,7 @@ func (s *Server) archiveStagedFile(
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
s.logger.Info("document staged, processing job queued",
|
||||
s.reqLog(ctx).Info("document staged, processing job queued",
|
||||
"document_id", doc.ID, "tenant_id", tenantID, "job_id", job.ID, "derive_title", deriveTitle)
|
||||
|
||||
// 7b. Eager thumbnail generation: same pipeline/cache path as the lazy
|
||||
@@ -1084,7 +1065,7 @@ func (s *Server) trySplitStagedUpload(
|
||||
|
||||
res, split, err := s.pagesplitter.Split(ctx, inboxPath)
|
||||
if err != nil {
|
||||
s.logger.Warn("separator-page split failed, archiving document unsplit",
|
||||
s.reqLog(ctx).Warn("separator-page split failed, archiving document unsplit",
|
||||
"tenant_id", tenantID, "filename", filename, "err", err)
|
||||
s.audlog.Log(audit.Entry{
|
||||
EventType: audit.EventDocumentSplit, Username: splitActor, TenantID: &tenantID, Success: false,
|
||||
@@ -1167,7 +1148,7 @@ func (s *Server) trySplitStagedUpload(
|
||||
EventType: audit.EventDocumentSplit, Username: splitActor, TenantID: &tenantID,
|
||||
DocumentID: strconv.FormatInt(docs[0].ID, 10), Success: true, Detail: detail,
|
||||
})
|
||||
s.logger.Info("upload split at barcode separator pages",
|
||||
s.reqLog(ctx).Info("upload split at barcode separator pages",
|
||||
"tenant_id", tenantID, "filename", filename, "pages", res.PageCount,
|
||||
"separator_pages", res.SeparatorPages, "parts", len(res.Parts),
|
||||
"documents_created", len(docs), "duplicates_skipped", dupCount)
|
||||
@@ -1238,7 +1219,7 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
|
||||
mimeType := detectMimeType("", ext, doc.StoragePath)
|
||||
result, err := s.ocr.Extract(ctx, doc.StoragePath, mimeType)
|
||||
if err != nil {
|
||||
s.logger.Warn("jobqueue ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID,
|
||||
s.reqLog(ctx).Warn("jobqueue ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID,
|
||||
"storage_path", doc.StoragePath, "resolved_mime", mimeType, "err", err)
|
||||
s.audlog.Log(audit.Entry{EventType: audit.EventDocumentProcessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "ocr_failed: " + err.Error()})
|
||||
return fmt.Errorf("jobqueue ocr extract: %w", err)
|
||||
@@ -1251,7 +1232,7 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
|
||||
// retry/backoff) never accumulates duplicates. Best-effort, never fails
|
||||
// the job since ocr_text is already the authoritative persisted result.
|
||||
if err := s.store.ReplaceOCRWords(ctx, documentID, ocrWordsFromResult(documentID, result.Words)); err != nil {
|
||||
s.logger.Warn("jobqueue replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
s.reqLog(ctx).Warn("jobqueue replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
}
|
||||
|
||||
if err := s.store.UpdateDocumentOCRText(ctx, documentID, tenantID, ocrText); err != nil {
|
||||
@@ -1267,7 +1248,7 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
|
||||
prefix, dateLayout := s.tenantScanTitleParams(ctx, tenantID)
|
||||
if newTitle := titleFromOCRText(ocrText, prefix, dateLayout); newTitle != "" && newTitle != doc.Title {
|
||||
if err := s.store.UpdateDocumentTitleAuto(ctx, documentID, tenantID, newTitle); err != nil {
|
||||
s.logger.Warn("jobqueue title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
s.reqLog(ctx).Warn("jobqueue title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
} else {
|
||||
doc.Title = newTitle
|
||||
}
|
||||
@@ -1286,7 +1267,7 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
|
||||
}
|
||||
if !sameDate(datePtr, doc.DocumentDate) {
|
||||
if err := s.store.UpdateDocumentDate(ctx, documentID, tenantID, datePtr, scorePtr); err != nil {
|
||||
s.logger.Warn("jobqueue document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
s.reqLog(ctx).Warn("jobqueue document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
} else {
|
||||
doc.DocumentDate = datePtr
|
||||
doc.DocumentDateScore = scorePtr
|
||||
@@ -1295,11 +1276,11 @@ func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID in
|
||||
}
|
||||
|
||||
if assignWarn := s.autoAssignTaxonomy(ctx, tenantID, doc, barcodes, ocrText); assignWarn != "" {
|
||||
s.logger.Info("jobqueue auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn)
|
||||
s.reqLog(ctx).Info("jobqueue auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn)
|
||||
}
|
||||
|
||||
if err := s.store.RunWorkflowsForDocument(ctx, tenantID, doc, storage.WorkflowTriggerOnUpload); err != nil {
|
||||
s.logger.Warn("jobqueue workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
s.reqLog(ctx).Warn("jobqueue workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
}
|
||||
|
||||
// Final index sync: auto-assignment/workflows may have re-indexed already,
|
||||
@@ -1323,7 +1304,7 @@ func (s *Server) autoAssignTaxonomy(ctx context.Context, tenantID int64, doc *st
|
||||
// Nachvollziehbarkeit even when nothing matched.
|
||||
if len(barcodes) > 0 {
|
||||
if err := s.store.SetDocumentBarcodeValues(ctx, doc.ID, tenantID, barcodes); err != nil {
|
||||
s.logger.Warn("failed to store barcode values", "document_id", doc.ID, "err", err)
|
||||
s.reqLog(ctx).Warn("failed to store barcode values", "document_id", doc.ID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1371,7 +1352,7 @@ func (s *Server) autoAssignTaxonomy(ctx context.Context, tenantID int64, doc *st
|
||||
for _, kind := range taxonomyKinds {
|
||||
entities, err := s.store.ListActiveMatchers(ctx, kind, tenantID)
|
||||
if err != nil {
|
||||
s.logger.Warn("failed to list active matchers", "kind", kind, "err", err)
|
||||
s.reqLog(ctx).Warn("failed to list active matchers", "kind", kind, "err", err)
|
||||
continue
|
||||
}
|
||||
for _, entity := range entities {
|
||||
@@ -1537,31 +1518,9 @@ func randomUploadID() string {
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
// copyFile is the cross-device fallback for os.Rename (EXDEV): copy + fsync
|
||||
// + remove the source.
|
||||
func copyFile(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o640)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
out.Close()
|
||||
os.Remove(dst)
|
||||
return err
|
||||
}
|
||||
if err := out.Sync(); err != nil {
|
||||
out.Close()
|
||||
os.Remove(dst)
|
||||
return err
|
||||
}
|
||||
return out.Close()
|
||||
}
|
||||
// The cross-device copy fallback for the WORM move lives in
|
||||
// internal/objectstore (copyFile there) since FDN-03 moved the archival step
|
||||
// behind the storage abstraction.
|
||||
|
||||
// ocrSupportedMimeTypes is the whitelist of MIME types the OCR pipeline
|
||||
// (internal/ocr.Extract) actually dispatches on. A declared Content-Type is
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
// FDN-08 — /metrics im Prometheus-Textformat, ohne Fremdabhängigkeit.
|
||||
//
|
||||
// Zugriff: bewusst OHNE Login (Scrape-Clients haben keine Session), dafür
|
||||
// IP-beschränkt. Default ist loopback-only; weitere Scraper werden über
|
||||
// config api.metrics_allowed_ips (IP oder CIDR) freigeschaltet. Es werden
|
||||
// ausschließlich aggregierte Zähler ausgegeben — keine Tenant-Daten, keine
|
||||
// Pfadsegmente mit IDs oder Tokens (siehe normalizeRoute).
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// handleMetrics rendert die Registry im Prometheus-Textformat.
|
||||
func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.metricsAllowed(r) {
|
||||
writeError(w, http.StatusForbidden, "metrics endpoint not allowed from this address")
|
||||
return
|
||||
}
|
||||
|
||||
routes, inFlight, panics := s.metrics.snapshot()
|
||||
|
||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
var b strings.Builder
|
||||
|
||||
fmt.Fprintf(&b, "# HELP archivdms_build_info Statische Build-Information.\n")
|
||||
fmt.Fprintf(&b, "# TYPE archivdms_build_info gauge\n")
|
||||
fmt.Fprintf(&b, "archivdms_build_info{version=\"%s\"} 1\n", escapeLabel(s.appVersion))
|
||||
|
||||
fmt.Fprintf(&b, "# HELP archivdms_uptime_seconds Laufzeit des Prozesses in Sekunden.\n")
|
||||
fmt.Fprintf(&b, "# TYPE archivdms_uptime_seconds gauge\n")
|
||||
fmt.Fprintf(&b, "archivdms_uptime_seconds %.3f\n", time.Since(s.startTime).Seconds())
|
||||
|
||||
fmt.Fprintf(&b, "# HELP archivdms_goroutines Aktuelle Anzahl Goroutinen.\n")
|
||||
fmt.Fprintf(&b, "# TYPE archivdms_goroutines gauge\n")
|
||||
fmt.Fprintf(&b, "archivdms_goroutines %d\n", runtime.NumGoroutine())
|
||||
|
||||
fmt.Fprintf(&b, "# HELP archivdms_http_requests_in_flight Aktuell laufende HTTP-Anfragen.\n")
|
||||
fmt.Fprintf(&b, "# TYPE archivdms_http_requests_in_flight gauge\n")
|
||||
fmt.Fprintf(&b, "archivdms_http_requests_in_flight %d\n", inFlight)
|
||||
|
||||
fmt.Fprintf(&b, "# HELP archivdms_panics_total Zentral abgefangene Panics (unbehandelte Fehler).\n")
|
||||
fmt.Fprintf(&b, "# TYPE archivdms_panics_total counter\n")
|
||||
fmt.Fprintf(&b, "archivdms_panics_total %d\n", panics)
|
||||
|
||||
// Requests + Latenz je Route/Status.
|
||||
keys := make([]routeKey, 0, len(routes))
|
||||
for k := range routes {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].route != keys[j].route {
|
||||
return keys[i].route < keys[j].route
|
||||
}
|
||||
if keys[i].method != keys[j].method {
|
||||
return keys[i].method < keys[j].method
|
||||
}
|
||||
return keys[i].status < keys[j].status
|
||||
})
|
||||
|
||||
fmt.Fprintf(&b, "# HELP archivdms_http_requests_total Anzahl HTTP-Anfragen je Route und Status.\n")
|
||||
fmt.Fprintf(&b, "# TYPE archivdms_http_requests_total counter\n")
|
||||
for _, k := range keys {
|
||||
st := routes[k]
|
||||
fmt.Fprintf(&b, "archivdms_http_requests_total{method=\"%s\",route=\"%s\",status=\"%d\"} %d\n",
|
||||
escapeLabel(k.method), escapeLabel(k.route), k.status, st.count)
|
||||
}
|
||||
|
||||
fmt.Fprintf(&b, "# HELP archivdms_http_request_duration_seconds Latenz der HTTP-Anfragen.\n")
|
||||
fmt.Fprintf(&b, "# TYPE archivdms_http_request_duration_seconds histogram\n")
|
||||
for _, k := range keys {
|
||||
st := routes[k]
|
||||
labels := fmt.Sprintf("method=\"%s\",route=\"%s\",status=\"%d\"",
|
||||
escapeLabel(k.method), escapeLabel(k.route), k.status)
|
||||
for i, ub := range latencyBuckets {
|
||||
fmt.Fprintf(&b, "archivdms_http_request_duration_seconds_bucket{%s,le=\"%g\"} %d\n",
|
||||
labels, ub, st.bucketCount[i])
|
||||
}
|
||||
fmt.Fprintf(&b, "archivdms_http_request_duration_seconds_bucket{%s,le=\"+Inf\"} %d\n", labels, st.count)
|
||||
fmt.Fprintf(&b, "archivdms_http_request_duration_seconds_sum{%s} %.6f\n", labels, st.sumSeconds)
|
||||
fmt.Fprintf(&b, "archivdms_http_request_duration_seconds_count{%s} %d\n", labels, st.count)
|
||||
}
|
||||
|
||||
// Queue-Länge (Akzeptanzkriterium 2). Fehler hier dürfen den Scrape nicht
|
||||
// scheitern lassen — dann fehlt die Metrik einfach für diesen Durchlauf.
|
||||
if s.store != nil {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
|
||||
defer cancel()
|
||||
counts, err := s.store.CountProcessingJobsByStatus(ctx)
|
||||
if err != nil {
|
||||
s.reqLog(r.Context()).Warn("metrics: queue length query failed", "err", err)
|
||||
} else {
|
||||
fmt.Fprintf(&b, "# HELP archivdms_processing_jobs Länge der Verarbeitungswarteschlange je Status.\n")
|
||||
fmt.Fprintf(&b, "# TYPE archivdms_processing_jobs gauge\n")
|
||||
statuses := make([]string, 0, len(counts))
|
||||
for st := range counts {
|
||||
statuses = append(statuses, st)
|
||||
}
|
||||
sort.Strings(statuses)
|
||||
for _, st := range statuses {
|
||||
fmt.Fprintf(&b, "archivdms_processing_jobs{status=\"%s\"} %d\n", escapeLabel(st), counts[st])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
w.Write([]byte(b.String()))
|
||||
}
|
||||
|
||||
// metricsAllowed prüft die Herkunft des Scrape-Requests: loopback immer,
|
||||
// sonst nur konfigurierte IPs/CIDRs (config api.metrics_allowed_ips).
|
||||
func (s *Server) metricsAllowed(r *http.Request) bool {
|
||||
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err != nil {
|
||||
host = r.RemoteAddr
|
||||
}
|
||||
ip := net.ParseIP(strings.TrimSpace(host))
|
||||
if ip == nil {
|
||||
return false
|
||||
}
|
||||
if ip.IsLoopback() {
|
||||
return true
|
||||
}
|
||||
for _, entry := range s.cfg.MetricsAllowedIPs {
|
||||
entry = strings.TrimSpace(entry)
|
||||
if entry == "" {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(entry, "/") {
|
||||
if _, cidr, err := net.ParseCIDR(entry); err == nil && cidr.Contains(ip) {
|
||||
return true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if parsed := net.ParseIP(entry); parsed != nil && parsed.Equal(ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// escapeLabel entschärft Anführungszeichen/Backslashes/Zeilenumbrüche in
|
||||
// Prometheus-Labelwerten.
|
||||
func escapeLabel(v string) string {
|
||||
r := strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`)
|
||||
return r.Replace(v)
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
// FDN-08 — Logging, Metriken & Fehler-Tracking.
|
||||
//
|
||||
// Dieses File enthält die drei Bausteine, die bisher gefehlt haben:
|
||||
//
|
||||
// 1. requestIDMiddleware: erzeugt (oder übernimmt aus X-Request-ID) eine
|
||||
// Korrelations-ID, hängt sie an den context.Context und gibt sie im
|
||||
// Response-Header zurück. Über loggerFromCtx(ctx) bekommt jede Log-Zeile
|
||||
// im Request-Lebenszyklus das Feld request_id, ohne dass jede Call-Site
|
||||
// umgeschrieben werden muss.
|
||||
// 2. metricsMiddleware: zählt Requests je (Methode, normalisierter Pfad,
|
||||
// Status) und summiert die Latenz in Histogramm-Buckets. Ausgabe über
|
||||
// GET /metrics im Prometheus-Textformat (kein prometheus/client_golang).
|
||||
// 3. recoverMiddleware: zentrales Panic-Recovery. net/http's ServeMux hat
|
||||
// keins; ohne das reißt ein Panic in einem Handler die Verbindung ab und
|
||||
// der Fehler taucht nirgends auf.
|
||||
//
|
||||
// Bewusst KEIN Logging von Query-Strings, Headern oder Request-Bodies:
|
||||
// dort stecken Tokens (Signed-URL-Signatur, Share-Token, Bearer-Keys) und
|
||||
// potenziell Passwörter. Geloggt werden ausschließlich Methode, normalisierter
|
||||
// Pfad (IDs/Tokens durch Platzhalter ersetzt), Status, Dauer und Client-IP.
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// stackTrace liefert einen gekürzten Stacktrace für das Panic-Log.
|
||||
func stackTrace() string {
|
||||
const maxStack = 4096
|
||||
buf := debug.Stack()
|
||||
if len(buf) > maxStack {
|
||||
buf = buf[:maxStack]
|
||||
}
|
||||
return string(buf)
|
||||
}
|
||||
|
||||
const (
|
||||
requestIDKey contextKey = "request_id"
|
||||
loggerKey contextKey = "logger"
|
||||
requestIDHeader = "X-Request-ID"
|
||||
maxRequestIDLen = 64
|
||||
)
|
||||
|
||||
// newRequestID erzeugt eine zufällige 16-stellige Hex-ID. Fällt bei einem
|
||||
// (praktisch unmöglichen) Fehler der Entropiequelle auf einen Zeitstempel
|
||||
// zurück — eine Anfrage darf daran nie scheitern.
|
||||
func newRequestID() string {
|
||||
buf := make([]byte, 8)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
return strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||
}
|
||||
return hex.EncodeToString(buf)
|
||||
}
|
||||
|
||||
// sanitizeRequestID übernimmt eine vom Client/Proxy gelieferte ID nur, wenn
|
||||
// sie kurz und druckbar-alphanumerisch ist. Verhindert Log-Injection über
|
||||
// Zeilenumbrüche im Header.
|
||||
func sanitizeRequestID(v string) string {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" || len(v) > maxRequestIDLen {
|
||||
return ""
|
||||
}
|
||||
for _, r := range v {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
|
||||
case r == '-' || r == '_' || r == '.':
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// requestIDFromCtx liefert die Korrelations-ID der laufenden Anfrage oder "".
|
||||
func requestIDFromCtx(ctx context.Context) string {
|
||||
if ctx == nil {
|
||||
return ""
|
||||
}
|
||||
v, _ := ctx.Value(requestIDKey).(string)
|
||||
return v
|
||||
}
|
||||
|
||||
// loggerFromCtx liefert den Request-Logger (inkl. request_id-Feld). Außerhalb
|
||||
// eines HTTP-Requests — oder wenn kein Logger hinterlegt wurde — kommt ein
|
||||
// no-op-freier Fallback zurück, damit Aufrufer nie auf nil prüfen müssen.
|
||||
func loggerFromCtx(ctx context.Context) *slog.Logger {
|
||||
if ctx != nil {
|
||||
if l, ok := ctx.Value(loggerKey).(*slog.Logger); ok && l != nil {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return slog.Default()
|
||||
}
|
||||
|
||||
// reqLog ist der Einstieg für Handler: nutzt den Request-Logger aus dem
|
||||
// Context, fällt aber auf den Server-Logger zurück, wenn die Middleware nicht
|
||||
// durchlaufen wurde (z.B. in Tests, die Handler direkt aufrufen).
|
||||
func (s *Server) reqLog(ctx context.Context) *slog.Logger {
|
||||
if ctx != nil {
|
||||
if l, ok := ctx.Value(loggerKey).(*slog.Logger); ok && l != nil {
|
||||
return l
|
||||
}
|
||||
}
|
||||
if s.logger != nil {
|
||||
return s.logger
|
||||
}
|
||||
return slog.Default()
|
||||
}
|
||||
|
||||
// requestIDMiddleware setzt die Korrelations-ID und den Request-Logger.
|
||||
func (s *Server) requestIDMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
rid := sanitizeRequestID(r.Header.Get(requestIDHeader))
|
||||
if rid == "" {
|
||||
rid = newRequestID()
|
||||
}
|
||||
base := s.logger
|
||||
if base == nil {
|
||||
base = slog.Default()
|
||||
}
|
||||
reqLogger := base.With("request_id", rid)
|
||||
|
||||
ctx := context.WithValue(r.Context(), requestIDKey, rid)
|
||||
ctx = context.WithValue(ctx, loggerKey, reqLogger)
|
||||
|
||||
w.Header().Set(requestIDHeader, rid)
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
}
|
||||
|
||||
// statusRecorder merkt sich Statuscode und geschriebene Bytes, damit die
|
||||
// Metrik-Middleware nach dem Handler auswerten kann.
|
||||
type statusRecorder struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
written int64
|
||||
wrote bool
|
||||
}
|
||||
|
||||
func (rec *statusRecorder) WriteHeader(code int) {
|
||||
if !rec.wrote {
|
||||
rec.status = code
|
||||
rec.wrote = true
|
||||
}
|
||||
rec.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (rec *statusRecorder) Write(b []byte) (int, error) {
|
||||
if !rec.wrote {
|
||||
rec.status = http.StatusOK
|
||||
rec.wrote = true
|
||||
}
|
||||
n, err := rec.ResponseWriter.Write(b)
|
||||
rec.written += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Flush reicht http.Flusher durch (Downloads/Streaming-Endpunkte).
|
||||
func (rec *statusRecorder) Flush() {
|
||||
if f, ok := rec.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
// recoverMiddleware fängt Panics zentral ab: Log mit Korrelations-ID +
|
||||
// Stacktrace, sauberer 500 an den Client. Ohne das stürzt zwar nicht der
|
||||
// Prozess (net/http fängt pro Verbindung ab), der Fehler bleibt aber
|
||||
// unsichtbar und der Client bekommt einen abgebrochenen Stream.
|
||||
func (s *Server) recoverMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
rec := recover()
|
||||
if rec == nil {
|
||||
return
|
||||
}
|
||||
if rec == http.ErrAbortHandler {
|
||||
panic(rec)
|
||||
}
|
||||
s.metrics.incPanic()
|
||||
loggerFromCtx(r.Context()).Error("panic in http handler",
|
||||
"method", r.Method,
|
||||
"path", normalizeRoute(r.URL.Path),
|
||||
"remote_ip", s.remoteIP(r),
|
||||
"panic", rec,
|
||||
"stack", stackTrace(),
|
||||
)
|
||||
if sr, ok := w.(*statusRecorder); ok && sr.wrote {
|
||||
return // Header sind raus, mehr geht nicht
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "interner Serverfehler")
|
||||
}()
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// metricsMiddleware misst Dauer und Status jeder Anfrage.
|
||||
func (s *Server) metricsMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
|
||||
start := time.Now()
|
||||
s.metrics.incInFlight()
|
||||
defer func() {
|
||||
s.metrics.decInFlight()
|
||||
d := time.Since(start)
|
||||
route := normalizeRoute(r.URL.Path)
|
||||
s.metrics.observe(r.Method, route, rec.status, d)
|
||||
// Zugriffs-Log: JEDE Anfrage erzeugt genau eine Zeile mit
|
||||
// request_id (AK1). Level nach Status gestaffelt, damit im
|
||||
// Normalbetrieb (Info) nur Auffälligkeiten sichtbar sind:
|
||||
// 5xx=Error, 4xx=Warn, Rest=Debug.
|
||||
lvl := slog.LevelDebug
|
||||
msg := "request completed"
|
||||
switch {
|
||||
case rec.status >= 500:
|
||||
lvl, msg = slog.LevelError, "request failed"
|
||||
case rec.status >= 400:
|
||||
lvl, msg = slog.LevelWarn, "request rejected"
|
||||
}
|
||||
loggerFromCtx(r.Context()).Log(r.Context(), lvl, msg,
|
||||
"method", r.Method, "route", route,
|
||||
"status", rec.status, "duration_ms", d.Milliseconds(),
|
||||
"bytes", rec.written,
|
||||
"remote_ip", s.remoteIP(r))
|
||||
}()
|
||||
next.ServeHTTP(rec, r)
|
||||
})
|
||||
}
|
||||
|
||||
// --- Pfad-Normalisierung ---
|
||||
|
||||
// tokenSegments sind Pfadabschnitte, deren FOLGENDES Segment ein Geheimnis
|
||||
// ist (Share-Token). Die dürfen niemals in Logs oder Metrik-Labels landen.
|
||||
var tokenSegments = map[string]bool{"share": true}
|
||||
|
||||
// normalizeRoute ersetzt variable Pfadsegmente durch Platzhalter. Das hält
|
||||
// die Label-Kardinalität der Metriken klein UND verhindert, dass IDs oder
|
||||
// Share-Tokens in Logs/Metriken auftauchen. Query-Strings werden nie
|
||||
// betrachtet (dort stehen Signaturen und API-Keys).
|
||||
func normalizeRoute(path string) string {
|
||||
if path == "" {
|
||||
return "/"
|
||||
}
|
||||
parts := strings.Split(path, "/")
|
||||
prevSecret := false
|
||||
for i, p := range parts {
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
if prevSecret {
|
||||
parts[i] = "{token}"
|
||||
prevSecret = false
|
||||
continue
|
||||
}
|
||||
prevSecret = tokenSegments[p]
|
||||
if isVariableSegment(p) {
|
||||
parts[i] = "{id}"
|
||||
}
|
||||
}
|
||||
out := strings.Join(parts, "/")
|
||||
if len(out) > 120 {
|
||||
return "/other"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// isVariableSegment erkennt IDs, Hashes und sonstige nicht-statische
|
||||
// Pfadbestandteile. Konservativ: im Zweifel maskieren.
|
||||
func isVariableSegment(seg string) bool {
|
||||
if _, err := strconv.ParseInt(seg, 10, 64); err == nil {
|
||||
return true
|
||||
}
|
||||
if len(seg) > 40 {
|
||||
return true
|
||||
}
|
||||
hasDigit := false
|
||||
for _, r := range seg {
|
||||
switch {
|
||||
case r >= '0' && r <= '9':
|
||||
hasDigit = true
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r == '-', r == '_', r == '.':
|
||||
default:
|
||||
// Alles Ungewöhnliche (Sonderzeichen, Umlaute, %-Encoding) ist mit
|
||||
// Sicherheit kein statisches Routensegment.
|
||||
return true
|
||||
}
|
||||
}
|
||||
// Statische Routensegmente sind reine Wörter ("delete-requests",
|
||||
// "classification-templates"). Längere Mischungen aus Buchstaben und
|
||||
// Ziffern sind Hashes/Tokens/Dateinamen -> maskieren.
|
||||
return hasDigit && len(seg) >= 8
|
||||
}
|
||||
|
||||
// --- Metrik-Registry ---
|
||||
|
||||
// latencyBuckets sind die oberen Grenzen (Sekunden) des Latenz-Histogramms.
|
||||
var latencyBuckets = []float64{0.005, 0.025, 0.1, 0.5, 1, 2.5, 5, 10, 30}
|
||||
|
||||
type routeKey struct {
|
||||
method string
|
||||
route string
|
||||
status int
|
||||
}
|
||||
|
||||
type routeStat struct {
|
||||
count uint64
|
||||
sumSeconds float64
|
||||
bucketCount []uint64 // len(latencyBuckets), kumulativ erst beim Rendern
|
||||
}
|
||||
|
||||
// metricsRegistry ist eine minimale, prozesslokale Metrik-Sammlung. Keine
|
||||
// globale Variable: hängt als Feld am Server (Dependency Injection).
|
||||
type metricsRegistry struct {
|
||||
mu sync.Mutex
|
||||
routes map[routeKey]*routeStat
|
||||
inFlight int64
|
||||
panics uint64
|
||||
started time.Time
|
||||
}
|
||||
|
||||
func newMetricsRegistry() *metricsRegistry {
|
||||
return &metricsRegistry{
|
||||
routes: make(map[routeKey]*routeStat),
|
||||
started: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *metricsRegistry) observe(method, route string, status int, d time.Duration) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
k := routeKey{method: method, route: route, status: status}
|
||||
secs := d.Seconds()
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
// Kardinalitätsbremse: unbekannte Pfade nicht unbegrenzt sammeln.
|
||||
st := m.routes[k]
|
||||
if st == nil {
|
||||
if len(m.routes) >= 500 {
|
||||
k = routeKey{method: method, route: "/other", status: status}
|
||||
st = m.routes[k]
|
||||
}
|
||||
if st == nil {
|
||||
st = &routeStat{bucketCount: make([]uint64, len(latencyBuckets))}
|
||||
m.routes[k] = st
|
||||
}
|
||||
}
|
||||
st.count++
|
||||
st.sumSeconds += secs
|
||||
for i, ub := range latencyBuckets {
|
||||
if secs <= ub {
|
||||
st.bucketCount[i]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *metricsRegistry) incInFlight() {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.inFlight++
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *metricsRegistry) decInFlight() {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.inFlight--
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *metricsRegistry) incPanic() {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.panics++
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
// snapshot liefert eine Kopie der Zähler für das Rendern.
|
||||
func (m *metricsRegistry) snapshot() (map[routeKey]routeStat, int64, uint64) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
out := make(map[routeKey]routeStat, len(m.routes))
|
||||
for k, v := range m.routes {
|
||||
cp := routeStat{count: v.count, sumSeconds: v.sumSeconds, bucketCount: make([]uint64, len(v.bucketCount))}
|
||||
copy(cp.bucketCount, v.bucketCount)
|
||||
out[k] = cp
|
||||
}
|
||||
return out, m.inFlight, m.panics
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
// FDN-08 — Tests zu den drei Akzeptanzkriterien:
|
||||
// 1. Korrelations-ID über alle Schichten
|
||||
// 2. Metriken (Latenz, Fehlerrate, Queue-Länge)
|
||||
// 3. Unbehandelte Fehler werden zentral gemeldet
|
||||
//
|
||||
// Zusätzlich Prüfung 2 der Abnahme: es dürfen keine Tokens/Passwörter in Logs
|
||||
// oder Metrik-Labels landen (TestNormalizeRouteRedactsSecrets).
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"archivdms/config"
|
||||
)
|
||||
|
||||
func newTestServer(buf *bytes.Buffer) *Server {
|
||||
logger := slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
|
||||
return New(config.APIConfig{}, nil, nil, nil, nil, logger)
|
||||
}
|
||||
|
||||
// AK 1: jede Anfrage bekommt eine Korrelations-ID, eine vom Client gelieferte
|
||||
// wird übernommen und im Response-Header zurückgegeben.
|
||||
func TestRequestIDGeneratedAndEchoed(t *testing.T) {
|
||||
srv := newTestServer(&bytes.Buffer{})
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/health", nil))
|
||||
got := rec.Header().Get(requestIDHeader)
|
||||
if got == "" {
|
||||
t.Fatalf("erwartete generierte Request-ID im Header %s", requestIDHeader)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/health", nil)
|
||||
req.Header.Set(requestIDHeader, "abc-123")
|
||||
rec2 := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec2, req)
|
||||
if rec2.Header().Get(requestIDHeader) != "abc-123" {
|
||||
t.Fatalf("Client-Request-ID nicht übernommen: %q", rec2.Header().Get(requestIDHeader))
|
||||
}
|
||||
|
||||
// Log-Injection: unsaubere IDs werden verworfen, nicht durchgereicht.
|
||||
bad := httptest.NewRequest(http.MethodGet, "/api/health", nil)
|
||||
bad.Header.Set(requestIDHeader, "evil\nid")
|
||||
rec3 := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec3, bad)
|
||||
if strings.Contains(rec3.Header().Get(requestIDHeader), "evil") {
|
||||
t.Fatalf("ungültige Request-ID wurde übernommen: %q", rec3.Header().Get(requestIDHeader))
|
||||
}
|
||||
}
|
||||
|
||||
// AK 1: die ID landet im Logger, den Handler über den Context ziehen.
|
||||
func TestLoggerFromCtxCarriesRequestID(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
srv := newTestServer(&buf)
|
||||
|
||||
h := srv.requestIDMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if requestIDFromCtx(r.Context()) == "" {
|
||||
t.Errorf("keine Request-ID im Context")
|
||||
}
|
||||
srv.reqLog(r.Context()).Info("testereignis")
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/health", nil))
|
||||
|
||||
rid := rec.Header().Get(requestIDHeader)
|
||||
if !strings.Contains(buf.String(), "request_id="+rid) {
|
||||
t.Fatalf("Log-Zeile ohne request_id=%s: %s", rid, buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// AK 2: Zähler und Latenz-Histogramm werden im Prometheus-Textformat geliefert.
|
||||
func TestMetricsEndpoint(t *testing.T) {
|
||||
srv := newTestServer(&bytes.Buffer{})
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
srv.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/api/health", nil))
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
req.RemoteAddr = "127.0.0.1:54321"
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("erwartet 200, bekam %d", rec.Code)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
want := `archivdms_http_requests_total{method="GET",route="/api/health",status="200"} 2`
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("Request-Zähler fehlt:\n%s", body)
|
||||
}
|
||||
for _, frag := range []string{
|
||||
"archivdms_http_request_duration_seconds_bucket",
|
||||
"archivdms_http_request_duration_seconds_sum",
|
||||
"archivdms_goroutines",
|
||||
"archivdms_panics_total",
|
||||
} {
|
||||
if !strings.Contains(body, frag) {
|
||||
t.Errorf("Metrik %q fehlt in der Ausgabe", frag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AK 2 (Zugriffsschutz): fremde IPs dürfen nicht scrapen.
|
||||
func TestMetricsEndpointIPRestricted(t *testing.T) {
|
||||
srv := newTestServer(&bytes.Buffer{})
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
req.RemoteAddr = "203.0.113.7:5000"
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("erwartet 403 für fremde IP, bekam %d", rec.Code)
|
||||
}
|
||||
|
||||
srv.cfg.MetricsAllowedIPs = []string{"203.0.113.0/24"}
|
||||
rec2 := httptest.NewRecorder()
|
||||
req2 := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
req2.RemoteAddr = "203.0.113.7:5000"
|
||||
srv.ServeHTTP(rec2, req2)
|
||||
if rec2.Code != http.StatusOK {
|
||||
t.Fatalf("erwartet 200 für freigeschaltete CIDR, bekam %d", rec2.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// AK 3: ein Panic wird zentral abgefangen, mit Korrelations-ID geloggt und als
|
||||
// sauberer 500 beantwortet — der Zähler archivdms_panics_total steigt.
|
||||
func TestRecoverMiddlewareCatchesPanic(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
srv := newTestServer(&buf)
|
||||
|
||||
boom := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
panic("kaputt")
|
||||
})
|
||||
h := srv.requestIDMiddleware(srv.metricsMiddleware(srv.recoverMiddleware(boom)))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/documents/42", nil))
|
||||
|
||||
if rec.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("erwartet 500, bekam %d", rec.Code)
|
||||
}
|
||||
rid := rec.Header().Get(requestIDHeader)
|
||||
logged := buf.String()
|
||||
if !strings.Contains(logged, "panic in http handler") || !strings.Contains(logged, "request_id="+rid) {
|
||||
t.Fatalf("Panic nicht mit Korrelations-ID geloggt: %s", logged)
|
||||
}
|
||||
if _, _, panics := srv.metrics.snapshot(); panics != 1 {
|
||||
t.Fatalf("erwartet 1 gezähltes Panic, bekam %d", panics)
|
||||
}
|
||||
}
|
||||
|
||||
// Abnahme-Prüfung 2: keine Tokens/IDs in Logs oder Metrik-Labels.
|
||||
func TestNormalizeRouteRedactsSecrets(t *testing.T) {
|
||||
cases := [][2]string{
|
||||
{"/api/health", "/api/health"},
|
||||
{"/api/documents/42", "/api/documents/{id}"},
|
||||
{"/api/documents/42/notes/7", "/api/documents/{id}/notes/{id}"},
|
||||
{"/api/classification-templates", "/api/classification-templates"},
|
||||
{"/api/trash/9/delete-requests", "/api/trash/{id}/delete-requests"},
|
||||
{"/public/share/s3cr3tTokenXyz", "/public/share/{token}"},
|
||||
{"/public/share/abc/download", "/public/share/{token}/download"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
in, want := c[0], c[1]
|
||||
if got := normalizeRoute(in); got != want {
|
||||
t.Errorf("normalizeRoute(%q) = %q, erwartet %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -51,7 +51,7 @@ func (s *Server) handleListDocumentOCRWords(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
words, err := s.store.ListOCRWords(r.Context(), id)
|
||||
if err != nil {
|
||||
s.logger.Error("list ocr words failed", "document_id", id, "tenant_id", *sess.TenantID, "err", err)
|
||||
s.reqLog(r.Context()).Error("list ocr words failed", "document_id", id, "tenant_id", *sess.TenantID, "err", err)
|
||||
writeError(w, http.StatusInternalServerError, "list ocr words failed")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -107,7 +106,7 @@ func (s *Server) handlePublicShareDownload(w http.ResponseWriter, r *http.Reques
|
||||
// Atomically claim one access slot (closes the max_accesses race).
|
||||
ok, err := s.store.IncrementShareAccess(r.Context(), rs.ShareID())
|
||||
if err != nil {
|
||||
s.logger.Error("share access increment failed", "share_id", rs.ShareID(), "err", err)
|
||||
s.reqLog(r.Context()).Error("share access increment failed", "share_id", rs.ShareID(), "err", err)
|
||||
writeError(w, http.StatusInternalServerError, "download failed")
|
||||
return
|
||||
}
|
||||
@@ -118,9 +117,9 @@ func (s *Server) handlePublicShareDownload(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
|
||||
f, err := os.Open(rs.StoragePath())
|
||||
f, err := s.objects.Open(r.Context(), rs.TenantID(), rs.StoragePath())
|
||||
if err != nil {
|
||||
s.logger.Error("share file open failed", "share_id", rs.ShareID(), "err", err)
|
||||
s.reqLog(r.Context()).Error("share file open failed", "share_id", rs.ShareID(), "err", err)
|
||||
writeError(w, http.StatusInternalServerError, "download failed")
|
||||
return
|
||||
}
|
||||
@@ -133,7 +132,7 @@ func (s *Server) handlePublicShareDownload(w http.ResponseWriter, r *http.Reques
|
||||
w.Header().Set("Content-Disposition", "attachment; filename=\""+safeDownloadName(rs.DocumentTitle, ext)+"\"")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
if _, err := io.Copy(w, f); err != nil {
|
||||
s.logger.Warn("share file stream interrupted", "share_id", rs.ShareID(), "err", err)
|
||||
s.reqLog(r.Context()).Warn("share file stream interrupted", "share_id", rs.ShareID(), "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -141,7 +140,7 @@ func (s *Server) handlePublicShareDownload(w http.ResponseWriter, r *http.Reques
|
||||
// into the audit log (EventShareAccessed). Never blocks the response path.
|
||||
func (s *Server) recordShareAccess(r *http.Request, rs *storage.ResolvedShare, ip, result string) {
|
||||
if err := s.store.LogShareAccess(r.Context(), rs.ShareID(), ip, r.UserAgent(), result); err != nil {
|
||||
s.logger.Error("share access log failed", "share_id", rs.ShareID(), "err", err)
|
||||
s.reqLog(r.Context()).Error("share access log failed", "share_id", rs.ShareID(), "err", err)
|
||||
}
|
||||
tenantID := rs.TenantID()
|
||||
s.audlog.Log(audit.Entry{
|
||||
|
||||
+83
-1
@@ -6,10 +6,13 @@ package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -19,6 +22,7 @@ import (
|
||||
"archivdms/internal/ldapauth"
|
||||
"archivdms/internal/ldapstore"
|
||||
"archivdms/internal/mailer"
|
||||
"archivdms/internal/objectstore"
|
||||
"archivdms/internal/ocr"
|
||||
"archivdms/internal/pagesplit"
|
||||
"archivdms/internal/storage"
|
||||
@@ -46,6 +50,10 @@ type Server struct {
|
||||
logger *slog.Logger
|
||||
mux *http.ServeMux
|
||||
ocr *ocr.Extractor
|
||||
// objects is the WORM object-storage driver (internal/objectstore): the
|
||||
// only place archived files are written, read or unlinked. Wired by
|
||||
// SetStorageConfig/SetObjectStore.
|
||||
objects objectstore.Store
|
||||
thumbs *thumbnail.Generator
|
||||
// pagesplitter performs barcode separator-page splitting of multi-page PDF
|
||||
// uploads before archival (internal/pagesplit). May be nil / disabled, in
|
||||
@@ -66,6 +74,15 @@ type Server struct {
|
||||
// (per client IP) to blunt token/password enumeration.
|
||||
shareLimiter *ipRateLimiter
|
||||
|
||||
// metrics ist die prozesslokale Metrik-Registry (FDN-08,
|
||||
// internal/api/observability.go). Kein globaler Zustand: hängt am Server.
|
||||
metrics *metricsRegistry
|
||||
|
||||
// baseHandler ist die in New() gebaute Middleware-Kette um s.mux
|
||||
// (requestID -> metrics -> recover). Einmal gebaut, danach nur gelesen —
|
||||
// kein Lazy-Init in ServeHTTP (Data Race).
|
||||
baseHandler http.Handler
|
||||
|
||||
// accountingLimiter rate-limits the Bearer-key Buchhaltungs-Pull-API
|
||||
// (per client IP) to blunt API-key guessing. Separate bucket set from
|
||||
// shareLimiter so a busy accounting client cannot starve share downloads.
|
||||
@@ -76,8 +93,46 @@ type Server struct {
|
||||
// paths, max upload size) into the API server. Needed by
|
||||
// handleUploadDocument, which cannot rely solely on the storage.Store
|
||||
// (that only knows its own base dir, not the inbox/ocr-tmp layout).
|
||||
//
|
||||
// It also constructs the object-storage driver (internal/objectstore), the
|
||||
// single place where archived files are written, read and unlinked. Call
|
||||
// SetFQDN before this if generated signed URLs should be absolute.
|
||||
func (s *Server) SetStorageConfig(cfg config.StorageConfig) {
|
||||
s.storageCfg = cfg
|
||||
secret := s.cfg.Secret
|
||||
if strings.TrimSpace(secret) == "" {
|
||||
// No master secret configured: fall back to an ephemeral, per-process
|
||||
// key so file access keeps working; signed URLs then simply do not
|
||||
// survive a restart. Never fail startup over this.
|
||||
buf := make([]byte, 32)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
secret = strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||
} else {
|
||||
secret = hex.EncodeToString(buf)
|
||||
}
|
||||
s.logger.Warn("objectstore: api.secret unset, using ephemeral signing key")
|
||||
}
|
||||
driver, err := objectstore.NewLocalStore(cfg, secret, s.publicBaseURL())
|
||||
if err != nil {
|
||||
s.logger.Error("objectstore init failed", "err", err)
|
||||
return
|
||||
}
|
||||
s.objects = driver
|
||||
}
|
||||
|
||||
// SetObjectStore overrides the object-storage driver (tests / alternative
|
||||
// wiring). Normally set implicitly by SetStorageConfig.
|
||||
func (s *Server) SetObjectStore(o objectstore.Store) {
|
||||
s.objects = o
|
||||
}
|
||||
|
||||
// publicBaseURL is the origin used for generated links. Empty FQDN yields
|
||||
// site-relative URLs.
|
||||
func (s *Server) publicBaseURL() string {
|
||||
if strings.TrimSpace(s.fqdn) == "" {
|
||||
return ""
|
||||
}
|
||||
return "https://" + strings.TrimSuffix(strings.TrimSpace(s.fqdn), "/")
|
||||
}
|
||||
|
||||
// SetOCR wires the OCR extractor into the API server. May be nil, in which
|
||||
@@ -150,8 +205,10 @@ func New(
|
||||
shareLimiter: newIPRateLimiter(20, 1.0),
|
||||
// Batch pulls are legitimate here: 60 requests burst, refilled at 5/sec.
|
||||
accountingLimiter: newIPRateLimiter(60, 5.0),
|
||||
metrics: newMetricsRegistry(),
|
||||
}
|
||||
s.routes()
|
||||
s.baseHandler = s.requestIDMiddleware(s.metricsMiddleware(s.recoverMiddleware(s.mux)))
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -168,6 +225,10 @@ func (s *Server) authAdmin(h http.HandlerFunc) http.HandlerFunc {
|
||||
func (s *Server) routes() {
|
||||
s.mux.HandleFunc("GET /api/health", s.handleHealth)
|
||||
s.mux.HandleFunc("GET /api/version", s.handleVersion)
|
||||
// Prometheus-Scrape-Endpunkt (FDN-08, internal/api/metrics_handlers.go).
|
||||
// Bewusst ohne s.auth — ein Scraper hat keine Session; der Zugriff wird
|
||||
// stattdessen per Quell-IP begrenzt (loopback + api.metrics_allowed_ips).
|
||||
s.mux.HandleFunc("GET /metrics", s.handleMetrics)
|
||||
|
||||
s.mux.HandleFunc("POST /api/auth/login", s.handleLogin)
|
||||
s.mux.HandleFunc("GET /api/auth/me", s.auth(s.handleMe))
|
||||
@@ -377,6 +438,12 @@ func (s *Server) routes() {
|
||||
s.mux.HandleFunc("GET /public/share/{token}", s.handlePublicShareMeta)
|
||||
s.mux.HandleFunc("POST /public/share/{token}/download", s.handlePublicShareDownload)
|
||||
|
||||
// Signed, time-limited download URLs (internal/api/signed_url_handlers.go).
|
||||
// Issuing is authenticated + tenant-scoped; redeeming runs WITHOUT s.auth
|
||||
// because the HMAC signature in the query string is the credential.
|
||||
s.mux.HandleFunc("POST /api/documents/{id}/signed-url", s.auth(s.handleCreateDocumentSignedURL))
|
||||
s.mux.HandleFunc("GET /public/files", s.handleSignedFileDownload)
|
||||
|
||||
// Buchhaltungs-Pull-API (internal/api/accounting_handlers.go).
|
||||
// Key administration runs on the normal session auth and is domain_admin-only
|
||||
// (a key grants tenant-wide read access to archived documents).
|
||||
@@ -417,8 +484,23 @@ func (s *Server) routes() {
|
||||
}
|
||||
|
||||
// ServeHTTP implements http.Handler.
|
||||
//
|
||||
// Die Basis-Middleware-Kette (FDN-08) liegt bewusst hier und nicht an den
|
||||
// einzelnen Routen, damit sie ausnahmslos für JEDE Anfrage gilt — auch für
|
||||
// /public/*, /metrics und nicht gefundene Pfade:
|
||||
//
|
||||
// requestID -> metrics -> recover -> ServeMux
|
||||
//
|
||||
// Reihenfolge: requestID zuerst, damit Metrik- und Panic-Log die
|
||||
// Korrelations-ID haben; recover innen, damit der 500 noch über den
|
||||
// statusRecorder der Metrik-Middleware läuft und dort gezählt wird.
|
||||
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
s.mux.ServeHTTP(w, r)
|
||||
if s.baseHandler == nil {
|
||||
// Server wurde nicht über New() gebaut (Tests): ohne Kette bedienen.
|
||||
s.mux.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
s.baseHandler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// --- system handlers ---
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
// Signed, time-limited download URLs for archived documents (FDN-03,
|
||||
// internal/objectstore). Same principle as the external share links
|
||||
// (share_handlers.go / public_share_handlers.go) — unguessable credential in
|
||||
// the link, mandatory hard expiry, IP rate limiting, audit trail — but
|
||||
// stateless: the credential is an HMAC-SHA256 signature over
|
||||
// tenant|document|expiry instead of a DB row.
|
||||
//
|
||||
// POST /api/documents/{id}/signed-url issue a link (authenticated, tenant-scoped)
|
||||
// GET /public/files?t=&d=&exp=&sig= redeem it (no session, signature is the credential)
|
||||
//
|
||||
// Use share links when a document is handed to an external party with its own
|
||||
// lifecycle (revoke, password, access cap); use signed URLs for short-lived
|
||||
// machine access to the file itself (viewer, export job) without a session
|
||||
// cookie. Nothing here bypasses tenant scoping: the tenant id is taken from
|
||||
// the signed payload and every lookup still filters on it.
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"archivdms/internal/audit"
|
||||
"archivdms/internal/objectstore"
|
||||
)
|
||||
|
||||
// createSignedURLRequest is the optional POST body. TTLMinutes overrides the
|
||||
// configured default validity (storage.signed_url_ttl_minutes); it is capped
|
||||
// at 24 hours so no effectively unbounded link can be minted.
|
||||
type createSignedURLRequest struct {
|
||||
TTLMinutes int `json:"ttl_minutes,omitempty"`
|
||||
}
|
||||
|
||||
// createSignedURLResponse is returned to the caller.
|
||||
type createSignedURLResponse struct {
|
||||
URL string `json:"url"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
|
||||
// maxSignedURLTTL caps a caller-supplied validity.
|
||||
const maxSignedURLTTL = 24 * time.Hour
|
||||
|
||||
// handleCreateDocumentSignedURL handles POST /api/documents/{id}/signed-url.
|
||||
func (s *Server) handleCreateDocumentSignedURL(w http.ResponseWriter, r *http.Request) {
|
||||
sess := sessionFromCtx(r.Context())
|
||||
if sess.TenantID == nil {
|
||||
writeError(w, http.StatusForbidden, "tenant context required")
|
||||
return
|
||||
}
|
||||
if s.objects == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "Dateispeicher nicht konfiguriert")
|
||||
return
|
||||
}
|
||||
docID, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid document id")
|
||||
return
|
||||
}
|
||||
var req createSignedURLRequest
|
||||
if r.Body != nil {
|
||||
_ = json.NewDecoder(r.Body).Decode(&req) // body is optional
|
||||
}
|
||||
ttl := time.Duration(req.TTLMinutes) * time.Minute
|
||||
if ttl <= 0 {
|
||||
ttl = s.storageCfg.ResolvedSignedURLTTL()
|
||||
}
|
||||
if ttl > maxSignedURLTTL {
|
||||
ttl = maxSignedURLTTL
|
||||
}
|
||||
|
||||
// IDOR guard: the document must belong to the caller's tenant. GetDocument
|
||||
// filters WHERE id = $1 AND tenant_id = $2.
|
||||
doc, err := s.store.GetDocument(r.Context(), docID, *sess.TenantID)
|
||||
if err != nil {
|
||||
s.logShare(r, audit.EventSignedURLCreated, sess.TenantID, sess.Username,
|
||||
"signed_url doc:"+strconv.FormatInt(docID, 10)+" err:not_found", false)
|
||||
writeError(w, http.StatusNotFound, "document not found")
|
||||
return
|
||||
}
|
||||
link, err := s.objects.SignedURL(*sess.TenantID, doc.ID, ttl)
|
||||
if err != nil {
|
||||
s.logShare(r, audit.EventSignedURLCreated, sess.TenantID, sess.Username,
|
||||
"signed_url doc:"+strconv.FormatInt(docID, 10)+" err:"+err.Error(), false)
|
||||
writeError(w, http.StatusInternalServerError, "Link konnte nicht erstellt werden")
|
||||
return
|
||||
}
|
||||
expiresAt := time.Now().Add(ttl)
|
||||
s.logShare(r, audit.EventSignedURLCreated, sess.TenantID, sess.Username,
|
||||
"signed_url doc:"+strconv.FormatInt(docID, 10)+" ttl:"+ttl.String(), true)
|
||||
writeJSON(w, http.StatusOK, createSignedURLResponse{URL: link, ExpiresAt: expiresAt})
|
||||
}
|
||||
|
||||
// handleSignedFileDownload handles GET /public/files?t=&d=&exp=&sig=. Served
|
||||
// WITHOUT the s.auth wrapper by design: the signature is the credential.
|
||||
// Order: rate-limit -> verify signature -> verify expiry -> tenant-scoped
|
||||
// document lookup -> stream from the WORM store. Every outcome is audit-logged
|
||||
// (EventSignedURLAccessed), successes and failures alike.
|
||||
func (s *Server) handleSignedFileDownload(w http.ResponseWriter, r *http.Request) {
|
||||
ip := s.remoteIP(r)
|
||||
if !s.shareLimiter.allow(ip) {
|
||||
writeError(w, http.StatusTooManyRequests, "too many requests")
|
||||
return
|
||||
}
|
||||
if s.objects == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "Dateispeicher nicht konfiguriert")
|
||||
return
|
||||
}
|
||||
ref, err := s.objects.VerifySignedURL(r.URL.Query(), time.Now())
|
||||
if err != nil {
|
||||
s.logSignedAccess(r, nil, "signed_url_access err:"+err.Error(), false)
|
||||
if errors.Is(err, objectstore.ErrSignatureExpired) {
|
||||
writeError(w, http.StatusGone, "Der Link ist abgelaufen.")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusForbidden, "Der Link ist ungültig.")
|
||||
return
|
||||
}
|
||||
|
||||
doc, err := s.store.GetDocument(r.Context(), ref.DocumentID, ref.TenantID)
|
||||
if err != nil {
|
||||
s.logSignedAccess(r, &ref.TenantID,
|
||||
"signed_url_access doc:"+strconv.FormatInt(ref.DocumentID, 10)+" err:not_found", false)
|
||||
writeError(w, http.StatusNotFound, "Dokument nicht gefunden.")
|
||||
return
|
||||
}
|
||||
f, err := s.objects.Open(r.Context(), ref.TenantID, doc.StoragePath)
|
||||
if err != nil {
|
||||
s.reqLog(r.Context()).Error("signed url file open failed", "document_id", doc.ID, "tenant_id", ref.TenantID, "err", err)
|
||||
s.logSignedAccess(r, &ref.TenantID,
|
||||
"signed_url_access doc:"+strconv.FormatInt(doc.ID, 10)+" err:"+err.Error(), false)
|
||||
if errors.Is(err, objectstore.ErrObjectNotFound) {
|
||||
writeError(w, http.StatusNotFound, "Datei nicht im Archiv vorhanden.")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "Download fehlgeschlagen.")
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
s.logSignedAccess(r, &ref.TenantID, "signed_url_access doc:"+strconv.FormatInt(doc.ID, 10), true)
|
||||
|
||||
ext := filepath.Ext(doc.StoragePath)
|
||||
w.Header().Set("Content-Type", detectMimeType("", ext, doc.StoragePath))
|
||||
w.Header().Set("Content-Disposition", "attachment; filename=\""+safeDownloadName(doc.Title, ext)+"\"")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
if _, err := io.Copy(w, f); err != nil {
|
||||
s.reqLog(r.Context()).Warn("signed url stream interrupted", "document_id", doc.ID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// logSignedAccess records a redemption attempt in the audit log. There is no
|
||||
// session here, so the username column carries the anonymous marker.
|
||||
func (s *Server) logSignedAccess(r *http.Request, tenantID *int64, detail string, ok bool) {
|
||||
s.audlog.Log(audit.Entry{
|
||||
EventType: audit.EventSignedURLAccessed, Username: "anonymous", TenantID: tenantID,
|
||||
IPAddress: s.remoteIP(r), Success: ok, Detail: detail,
|
||||
})
|
||||
}
|
||||
@@ -87,6 +87,12 @@ const (
|
||||
EventShareRevoked = "share_revoked"
|
||||
EventShareAccessed = "share_accessed"
|
||||
|
||||
// Signed, time-limited storage download URLs (internal/objectstore,
|
||||
// internal/api/signed_url_handlers.go). Created is logged when a link is
|
||||
// issued, Accessed on every attempt to redeem one (success and failure).
|
||||
EventSignedURLCreated = "signed_url_created"
|
||||
EventSignedURLAccessed = "signed_url_accessed"
|
||||
|
||||
// LDAP directory integration (internal/ldapstore, internal/ldapauth,
|
||||
// internal/api/ldap_handlers.go). ConfigChanged covers create/update/delete
|
||||
// and the test action; LoginSuccess/Failed are logged on every LDAP bind
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/hkdf"
|
||||
|
||||
"archivdms/config"
|
||||
)
|
||||
|
||||
// signKeyInfo domain-separates the download-URL signing key from every other
|
||||
// key derived from the same master secret (JWT uses "archivdms-jwt-v1", the
|
||||
// LDAP secretbox uses "archivdms-ldap-secretbox-v1").
|
||||
const signKeyInfo = "archivdms-storage-url-v1"
|
||||
|
||||
// SignedPath is the route a signed download URL points at. It is served
|
||||
// WITHOUT session auth — the signature is the credential.
|
||||
const SignedPath = "/public/files"
|
||||
|
||||
// LocalStore is the local-filesystem WORM driver and the only implementation
|
||||
// of Store. It owns no state beyond the storage configuration, the URL signing
|
||||
// key and the public base URL used for link generation.
|
||||
type LocalStore struct {
|
||||
cfg config.StorageConfig
|
||||
signKey []byte
|
||||
baseURL string
|
||||
}
|
||||
|
||||
// compile-time interface check.
|
||||
var _ Store = (*LocalStore)(nil)
|
||||
|
||||
// NewLocalStore builds the local driver. secret is the application master
|
||||
// secret (config.APIConfig.Secret) from which the URL signing key is derived
|
||||
// via HKDF-SHA256; baseURL is the public origin used for generated links
|
||||
// (empty = emit site-relative URLs).
|
||||
func NewLocalStore(cfg config.StorageConfig, secret, baseURL string) (*LocalStore, error) {
|
||||
if strings.TrimSpace(secret) == "" {
|
||||
return nil, fmt.Errorf("objectstore: empty signing secret")
|
||||
}
|
||||
key := make([]byte, 32)
|
||||
if _, err := io.ReadFull(hkdf.New(sha256.New, []byte(secret), nil, []byte(signKeyInfo)), key); err != nil {
|
||||
return nil, fmt.Errorf("objectstore: derive signing key: %w", err)
|
||||
}
|
||||
return &LocalStore{
|
||||
cfg: cfg,
|
||||
signKey: key,
|
||||
baseURL: strings.TrimRight(strings.TrimSpace(baseURL), "/"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// tenantRoot is the tenant's WORM subtree: <BasePath>/store/<tenant_id>.
|
||||
func (l *LocalStore) tenantRoot(tenantID int64) string {
|
||||
return filepath.Join(l.cfg.StorePath(), strconv.FormatInt(tenantID, 10))
|
||||
}
|
||||
|
||||
// resolveTenantPath cleans storagePath and verifies it lies inside the
|
||||
// tenant's own store subtree. This is the filesystem-level counterpart of the
|
||||
// "WHERE tenant_id = $N" rule: even a manipulated storage_path from the DB
|
||||
// cannot be used to read another tenant's archive.
|
||||
func (l *LocalStore) resolveTenantPath(tenantID int64, storagePath string) (string, error) {
|
||||
if strings.TrimSpace(storagePath) == "" {
|
||||
return "", ErrObjectNotFound
|
||||
}
|
||||
clean := filepath.Clean(storagePath)
|
||||
root := filepath.Clean(l.tenantRoot(tenantID))
|
||||
if clean != root && !strings.HasPrefix(clean, root+string(os.PathSeparator)) {
|
||||
return "", ErrOutsideTenant
|
||||
}
|
||||
return clean, nil
|
||||
}
|
||||
|
||||
// Archive implements Store. It reproduces, unchanged, the archival steps the
|
||||
// upload pipeline has always performed: build store/<tenant>/<yyyy>/<mm>,
|
||||
// reject an existing target as duplicate, move (rename, copy+remove fallback
|
||||
// across devices) and finally chmod 0440 — the one and only chmod, applied
|
||||
// once the file sits at its final path.
|
||||
func (l *LocalStore) Archive(ctx context.Context, tenantID int64, srcPath, ext, contentHash string, at time.Time) (string, error) {
|
||||
if at.IsZero() {
|
||||
at = time.Now()
|
||||
}
|
||||
storeDir := filepath.Join(l.tenantRoot(tenantID),
|
||||
fmt.Sprintf("%04d", at.Year()), fmt.Sprintf("%02d", at.Month()))
|
||||
if err := os.MkdirAll(storeDir, 0o750); err != nil {
|
||||
os.Remove(srcPath)
|
||||
return "", fmt.Errorf("objectstore: create store dir: %w", err)
|
||||
}
|
||||
dst := filepath.Join(storeDir, contentHash+ext)
|
||||
|
||||
// Collision check: identical hash already stored -> duplicate.
|
||||
if _, err := os.Stat(dst); err == nil {
|
||||
os.Remove(srcPath)
|
||||
return "", ErrObjectExists
|
||||
} else if !os.IsNotExist(err) {
|
||||
os.Remove(srcPath)
|
||||
return "", fmt.Errorf("objectstore: stat store path: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Rename(srcPath, dst); err != nil {
|
||||
if copyErr := copyFile(srcPath, dst); copyErr != nil {
|
||||
os.Remove(srcPath)
|
||||
return "", fmt.Errorf("objectstore: move file to store: rename failed (%v), copy fallback failed: %w", err, copyErr)
|
||||
}
|
||||
os.Remove(srcPath)
|
||||
}
|
||||
|
||||
// WORM lock: read-only for everyone from now on.
|
||||
if err := os.Chmod(dst, 0o440); err != nil {
|
||||
return "", fmt.Errorf("objectstore: chmod store file: %w", err)
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
// Open implements Store.
|
||||
func (l *LocalStore) Open(ctx context.Context, tenantID int64, storagePath string) (io.ReadSeekCloser, error) {
|
||||
p, err := l.resolveTenantPath(tenantID, storagePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := os.Open(p)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("objectstore: open %q: %w", p, ErrObjectNotFound)
|
||||
}
|
||||
return nil, fmt.Errorf("objectstore: open object: %w", err)
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Stat implements Store.
|
||||
func (l *LocalStore) Stat(ctx context.Context, tenantID int64, storagePath string) (os.FileInfo, error) {
|
||||
p, err := l.resolveTenantPath(tenantID, storagePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fi, err := os.Stat(p)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("objectstore: stat %q: %w", p, ErrObjectNotFound)
|
||||
}
|
||||
return nil, fmt.Errorf("objectstore: stat object: %w", err)
|
||||
}
|
||||
return fi, nil
|
||||
}
|
||||
|
||||
// Delete implements Store. Only legitimate after a confirmed deletion request
|
||||
// whose retention period has expired — this layer performs no retention check
|
||||
// of its own; that stays in storage.ConfirmDeleteRequest, which still unlinks
|
||||
// inside its own transaction and is deliberately left untouched by FDN-03
|
||||
// (moving it here would mean handing the DB layer a filesystem driver).
|
||||
func (l *LocalStore) Delete(ctx context.Context, tenantID int64, storagePath string) error {
|
||||
p, err := l.resolveTenantPath(tenantID, storagePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(p); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return fmt.Errorf("objectstore: remove %q: %w", p, ErrObjectNotFound)
|
||||
}
|
||||
return fmt.Errorf("objectstore: remove object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SignedURL implements Store. The URL carries tenant id, document id and an
|
||||
// absolute expiry, authenticated by an HMAC-SHA256 over exactly those three
|
||||
// values — the same "unguessable token, hard expiry, server-side check"
|
||||
// principle as the external share links, only stateless (no DB row).
|
||||
func (l *LocalStore) SignedURL(tenantID, documentID int64, ttl time.Duration) (string, error) {
|
||||
if tenantID <= 0 || documentID <= 0 {
|
||||
return "", fmt.Errorf("objectstore: invalid signed url reference")
|
||||
}
|
||||
if ttl <= 0 {
|
||||
ttl = l.cfg.ResolvedSignedURLTTL()
|
||||
}
|
||||
exp := time.Now().Add(ttl).Unix()
|
||||
q := url.Values{}
|
||||
q.Set("t", strconv.FormatInt(tenantID, 10))
|
||||
q.Set("d", strconv.FormatInt(documentID, 10))
|
||||
q.Set("exp", strconv.FormatInt(exp, 10))
|
||||
q.Set("sig", l.sign(tenantID, documentID, exp))
|
||||
return l.baseURL + SignedPath + "?" + q.Encode(), nil
|
||||
}
|
||||
|
||||
// VerifySignedURL implements Store. Signature first, expiry second, so a
|
||||
// forged link never learns anything from the expiry branch.
|
||||
func (l *LocalStore) VerifySignedURL(q url.Values, now time.Time) (SignedRef, error) {
|
||||
tenantID, err1 := strconv.ParseInt(q.Get("t"), 10, 64)
|
||||
documentID, err2 := strconv.ParseInt(q.Get("d"), 10, 64)
|
||||
exp, err3 := strconv.ParseInt(q.Get("exp"), 10, 64)
|
||||
sig := q.Get("sig")
|
||||
if err1 != nil || err2 != nil || err3 != nil || sig == "" || tenantID <= 0 || documentID <= 0 {
|
||||
return SignedRef{}, ErrSignatureInvalid
|
||||
}
|
||||
want := l.sign(tenantID, documentID, exp)
|
||||
if !hmac.Equal([]byte(want), []byte(sig)) {
|
||||
return SignedRef{}, ErrSignatureInvalid
|
||||
}
|
||||
expiresAt := time.Unix(exp, 0)
|
||||
if !now.Before(expiresAt) {
|
||||
return SignedRef{}, ErrSignatureExpired
|
||||
}
|
||||
return SignedRef{TenantID: tenantID, DocumentID: documentID, ExpiresAt: expiresAt}, nil
|
||||
}
|
||||
|
||||
// sign returns the base64url HMAC-SHA256 over the canonical payload.
|
||||
func (l *LocalStore) sign(tenantID, documentID, exp int64) string {
|
||||
mac := hmac.New(sha256.New, l.signKey)
|
||||
fmt.Fprintf(mac, "v1|%d|%d|%d", tenantID, documentID, exp)
|
||||
return base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
// copyFile is the cross-device fallback for os.Rename (EXDEV): copy + fsync.
|
||||
// The source is removed by the caller.
|
||||
func copyFile(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o640)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
out.Close()
|
||||
os.Remove(dst)
|
||||
return err
|
||||
}
|
||||
if err := out.Sync(); err != nil {
|
||||
out.Close()
|
||||
os.Remove(dst)
|
||||
return err
|
||||
}
|
||||
return out.Close()
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"archivdms/config"
|
||||
)
|
||||
|
||||
func newTestStore(t *testing.T) *LocalStore {
|
||||
t.Helper()
|
||||
l, err := NewLocalStore(config.StorageConfig{BasePath: t.TempDir()}, "test-master-secret", "https://dms.example.test")
|
||||
if err != nil {
|
||||
t.Fatalf("NewLocalStore: %v", err)
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// stage writes a scratch file and returns its path plus content hash.
|
||||
func stage(t *testing.T, content string) (string, string) {
|
||||
t.Helper()
|
||||
p := filepath.Join(t.TempDir(), "scratch.pdf")
|
||||
if err := os.WriteFile(p, []byte(content), 0o640); err != nil {
|
||||
t.Fatalf("write scratch: %v", err)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(content))
|
||||
return p, hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// AK 1 + Prüfung 1: Round-Trip Archive -> Open, WORM path scheme and 0440.
|
||||
func TestArchiveOpenRoundTrip(t *testing.T) {
|
||||
l := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
src, hash := stage(t, "hello worm")
|
||||
at := time.Date(2026, 3, 7, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
dst, err := l.Archive(ctx, 42, src, ".pdf", hash, at)
|
||||
if err != nil {
|
||||
t.Fatalf("Archive: %v", err)
|
||||
}
|
||||
want := filepath.Join(l.cfg.StorePath(), "42", "2026", "03", hash+".pdf")
|
||||
if dst != want {
|
||||
t.Fatalf("path scheme changed: got %q want %q", dst, want)
|
||||
}
|
||||
fi, err := os.Stat(dst)
|
||||
if err != nil {
|
||||
t.Fatalf("stat archived: %v", err)
|
||||
}
|
||||
if fi.Mode().Perm() != 0o440 {
|
||||
t.Fatalf("WORM permissions: got %o want 0440", fi.Mode().Perm())
|
||||
}
|
||||
if _, err := os.Stat(src); !os.IsNotExist(err) {
|
||||
t.Fatalf("scratch file not consumed")
|
||||
}
|
||||
|
||||
f, err := l.Open(ctx, 42, dst)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
got, _ := io.ReadAll(f)
|
||||
if string(got) != "hello worm" {
|
||||
t.Fatalf("round-trip content mismatch: %q", got)
|
||||
}
|
||||
|
||||
// Duplicate archival of the same content is rejected.
|
||||
src2, _ := stage(t, "hello worm")
|
||||
if _, err := l.Archive(ctx, 42, src2, ".pdf", hash, at); !errors.Is(err, ErrObjectExists) {
|
||||
t.Fatalf("duplicate: got %v want ErrObjectExists", err)
|
||||
}
|
||||
}
|
||||
|
||||
// AK 1: cross-tenant access is refused even with a valid path.
|
||||
func TestOpenForeignTenantRejected(t *testing.T) {
|
||||
l := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
src, hash := stage(t, "tenant one")
|
||||
dst, err := l.Archive(ctx, 1, src, ".pdf", hash, time.Now())
|
||||
if err != nil {
|
||||
t.Fatalf("Archive: %v", err)
|
||||
}
|
||||
if _, err := l.Open(ctx, 2, dst); !errors.Is(err, ErrOutsideTenant) {
|
||||
t.Fatalf("foreign tenant: got %v want ErrOutsideTenant", err)
|
||||
}
|
||||
if _, err := l.Open(ctx, 1, filepath.Join(filepath.Dir(dst), "..", "..", "..", "2", "x.pdf")); !errors.Is(err, ErrOutsideTenant) {
|
||||
t.Fatalf("traversal: got %v want ErrOutsideTenant", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Prüfung 3: missing object yields a clear, typed error.
|
||||
func TestMissingObjectErrors(t *testing.T) {
|
||||
l := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
missing := filepath.Join(l.cfg.StorePath(), "7", "2026", "01", "deadbeef.pdf")
|
||||
|
||||
if _, err := l.Open(ctx, 7, missing); !errors.Is(err, ErrObjectNotFound) {
|
||||
t.Fatalf("Open: got %v want ErrObjectNotFound", err)
|
||||
}
|
||||
if _, err := l.Stat(ctx, 7, missing); !errors.Is(err, ErrObjectNotFound) {
|
||||
t.Fatalf("Stat: got %v want ErrObjectNotFound", err)
|
||||
}
|
||||
if err := l.Delete(ctx, 7, missing); !errors.Is(err, ErrObjectNotFound) {
|
||||
t.Fatalf("Delete: got %v want ErrObjectNotFound", err)
|
||||
}
|
||||
if _, err := l.Open(ctx, 7, ""); !errors.Is(err, ErrObjectNotFound) {
|
||||
t.Fatalf("empty path: got %v want ErrObjectNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteRemovesObject(t *testing.T) {
|
||||
l := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
src, hash := stage(t, "to be deleted")
|
||||
dst, err := l.Archive(ctx, 5, src, ".pdf", hash, time.Now())
|
||||
if err != nil {
|
||||
t.Fatalf("Archive: %v", err)
|
||||
}
|
||||
if err := l.Delete(ctx, 5, dst); err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(dst); !os.IsNotExist(err) {
|
||||
t.Fatalf("object still present after delete")
|
||||
}
|
||||
}
|
||||
|
||||
// AK 2 + Prüfung 2: signed URLs verify while valid and are refused afterwards.
|
||||
func TestSignedURLLifecycle(t *testing.T) {
|
||||
l := newTestStore(t)
|
||||
link, err := l.SignedURL(3, 99, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("SignedURL: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(link, "https://dms.example.test"+SignedPath+"?") {
|
||||
t.Fatalf("unexpected link: %s", link)
|
||||
}
|
||||
u, err := url.Parse(link)
|
||||
if err != nil {
|
||||
t.Fatalf("parse link: %v", err)
|
||||
}
|
||||
ref, err := l.VerifySignedURL(u.Query(), time.Now())
|
||||
if err != nil {
|
||||
t.Fatalf("verify: %v", err)
|
||||
}
|
||||
if ref.TenantID != 3 || ref.DocumentID != 99 {
|
||||
t.Fatalf("payload mismatch: %+v", ref)
|
||||
}
|
||||
|
||||
// Expired.
|
||||
if _, err := l.VerifySignedURL(u.Query(), time.Now().Add(2*time.Minute)); !errors.Is(err, ErrSignatureExpired) {
|
||||
t.Fatalf("expired: got %v want ErrSignatureExpired", err)
|
||||
}
|
||||
|
||||
// Tampered document id.
|
||||
q := u.Query()
|
||||
q.Set("d", "100")
|
||||
if _, err := l.VerifySignedURL(q, time.Now()); !errors.Is(err, ErrSignatureInvalid) {
|
||||
t.Fatalf("tampered: got %v want ErrSignatureInvalid", err)
|
||||
}
|
||||
|
||||
// Foreign key material.
|
||||
other, err := NewLocalStore(config.StorageConfig{BasePath: t.TempDir()}, "different-secret", "")
|
||||
if err != nil {
|
||||
t.Fatalf("NewLocalStore: %v", err)
|
||||
}
|
||||
if _, err := other.VerifySignedURL(u.Query(), time.Now()); !errors.Is(err, ErrSignatureInvalid) {
|
||||
t.Fatalf("foreign key: got %v want ErrSignatureInvalid", err)
|
||||
}
|
||||
|
||||
// Missing parameters.
|
||||
if _, err := l.VerifySignedURL(url.Values{}, time.Now()); !errors.Is(err, ErrSignatureInvalid) {
|
||||
t.Fatalf("empty query: got %v want ErrSignatureInvalid", err)
|
||||
}
|
||||
}
|
||||
|
||||
// AK 2: ttl <= 0 falls back to the configured default validity.
|
||||
func TestSignedURLDefaultTTL(t *testing.T) {
|
||||
l, err := NewLocalStore(config.StorageConfig{BasePath: t.TempDir(), SignedURLTTLMinutes: 5}, "s", "")
|
||||
if err != nil {
|
||||
t.Fatalf("NewLocalStore: %v", err)
|
||||
}
|
||||
link, err := l.SignedURL(1, 1, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("SignedURL: %v", err)
|
||||
}
|
||||
u, _ := url.Parse(link)
|
||||
if strings.HasPrefix(link, "http") {
|
||||
t.Fatalf("empty baseURL must yield a relative link: %s", link)
|
||||
}
|
||||
if _, err := l.VerifySignedURL(u.Query(), time.Now().Add(4*time.Minute)); err != nil {
|
||||
t.Fatalf("within default ttl: %v", err)
|
||||
}
|
||||
if _, err := l.VerifySignedURL(u.Query(), time.Now().Add(6*time.Minute)); !errors.Is(err, ErrSignatureExpired) {
|
||||
t.Fatalf("past default ttl: got %v want ErrSignatureExpired", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
// Package objectstore puts the existing local WORM document storage behind a
|
||||
// small Go interface (FDN-03). It is a pure abstraction layer: the on-disk
|
||||
// layout, the chmod 0440 WORM lock and the SHA-256 content addressing are
|
||||
// exactly the ones the upload pipeline has always used — nothing about the
|
||||
// path scheme or the archival semantics changes here.
|
||||
//
|
||||
// # Pfadschema (bestehend, NICHT verändert)
|
||||
//
|
||||
// All paths are rooted at config.Storage.BasePath (default /var/lib/archivdms):
|
||||
//
|
||||
// <BasePath>/inbox/<tenant_id>/<random>.<ext> raw upload, scratch, writable (0640)
|
||||
// <BasePath>/store/<tenant_id>/<yyyy>/<mm>/<sha256>.<ext> finished archive, WORM (0440)
|
||||
// <BasePath>/ocr-tmp/<random>/ OCR scratch, removed after use
|
||||
// <BasePath>/thumbnails/<tenant_id>/<sha256>.png regenerable preview, not WORM
|
||||
//
|
||||
// Properties of the store/ layer that callers may rely on:
|
||||
//
|
||||
// - Tenant separation is the FIRST path segment: every object of a tenant
|
||||
// lives below store/<tenant_id>/ and nowhere else. Open/Stat/Delete
|
||||
// therefore verify that the given path really is inside that tenant's
|
||||
// subtree (containment check) — a stored path from a foreign tenant is
|
||||
// rejected with ErrOutsideTenant instead of being read.
|
||||
// - <yyyy>/<mm> is derived from the archival (upload) time, not from the
|
||||
// recognised Belegdatum: after the WORM move a file is never moved again.
|
||||
// - The file name is the lowercase hex SHA-256 of the file content plus the
|
||||
// original extension. Content addressing gives byte-identical re-uploads
|
||||
// the same path, which is the filesystem half of the duplicate protection
|
||||
// (the DB unique index on (tenant_id, content_hash) is the other half).
|
||||
// - Archived files are chmod 0440. The directory stays writable for the
|
||||
// service user, so a legally confirmed deletion (after retain_until) can
|
||||
// still unlink the file — no code path ever overwrites an archived file.
|
||||
// - Nothing is encrypted or container-wrapped: every object is readable with
|
||||
// plain OS tools, deliberately unlike a closed vendor archive.
|
||||
//
|
||||
// Deliberately NO S3/object-storage driver: the WORM/GoBD guarantee rests on
|
||||
// POSIX file permissions (0440) which an object store cannot provide in the
|
||||
// same way. The local driver is and stays the only implementation.
|
||||
package objectstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Storage-level errors. Callers map these onto HTTP status codes / domain
|
||||
// errors (e.g. ErrObjectExists -> storage.ErrDuplicateContentHash).
|
||||
var (
|
||||
// ErrObjectExists is returned by Archive when the target WORM path is
|
||||
// already taken, i.e. the identical content is already archived.
|
||||
ErrObjectExists = errors.New("objectstore: object already exists")
|
||||
// ErrObjectNotFound is returned by Open/Stat/Delete when the object does
|
||||
// not exist on disk.
|
||||
ErrObjectNotFound = errors.New("objectstore: object not found")
|
||||
// ErrOutsideTenant is returned when a storage path does not resolve into
|
||||
// the requesting tenant's store subtree (IDOR / path-traversal guard).
|
||||
ErrOutsideTenant = errors.New("objectstore: path outside tenant store")
|
||||
// ErrSignatureInvalid is returned when a signed URL is malformed or its
|
||||
// HMAC does not verify.
|
||||
ErrSignatureInvalid = errors.New("objectstore: signature invalid")
|
||||
// ErrSignatureExpired is returned when a signed URL's expiry has passed.
|
||||
ErrSignatureExpired = errors.New("objectstore: signature expired")
|
||||
)
|
||||
|
||||
// SignedRef is the payload carried by a signed download URL: which document of
|
||||
// which tenant may be downloaded, and until when.
|
||||
type SignedRef struct {
|
||||
TenantID int64
|
||||
DocumentID int64
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
// Store is the document blob storage abstraction. Every method is
|
||||
// tenant-scoped; there is intentionally no "list everything" call.
|
||||
type Store interface {
|
||||
// Archive moves an already-hashed scratch file (inbox or split part) into
|
||||
// the tenant's WORM store and locks it with chmod 0440. It returns the
|
||||
// final storage path. On success the caller no longer owns srcPath; on
|
||||
// failure srcPath is removed. Returns ErrObjectExists when the content is
|
||||
// already archived (duplicate).
|
||||
Archive(ctx context.Context, tenantID int64, srcPath, ext, contentHash string, at time.Time) (string, error)
|
||||
|
||||
// Open opens an archived object read-only after verifying that
|
||||
// storagePath belongs to tenantID.
|
||||
Open(ctx context.Context, tenantID int64, storagePath string) (io.ReadSeekCloser, error)
|
||||
|
||||
// Stat reports metadata of an archived object (tenant-checked).
|
||||
Stat(ctx context.Context, tenantID int64, storagePath string) (os.FileInfo, error)
|
||||
|
||||
// Delete unlinks an archived object (tenant-checked). Only ever called
|
||||
// after a confirmed, retention-cleared deletion request; a missing file is
|
||||
// reported as ErrObjectNotFound.
|
||||
Delete(ctx context.Context, tenantID int64, storagePath string) error
|
||||
|
||||
// SignedURL builds a time-limited, HMAC-signed download URL for a
|
||||
// document. ttl <= 0 uses the configured default validity.
|
||||
SignedURL(tenantID, documentID int64, ttl time.Duration) (string, error)
|
||||
|
||||
// VerifySignedURL validates the query parameters of a signed URL against
|
||||
// the signing key and the current time.
|
||||
VerifySignedURL(q url.Values, now time.Time) (SignedRef, error)
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
-- 024_ocr_words.down.sql
|
||||
-- Rollback documentation for 024_ocr_words.sql.
|
||||
-- Documentation only — archivdms has no migration runner; run manually via
|
||||
-- psql after deploying a binary whose initSchema no longer creates the table.
|
||||
--
|
||||
-- PRECONDITION: Store.initOCRWordsSchema must be removed from
|
||||
-- Store.initSchema (internal/storage/documents.go) and every caller of
|
||||
-- ReplaceOCRWords / the word-box read path must be gone first — otherwise
|
||||
-- the next process start recreates the table.
|
||||
--
|
||||
-- DATA LOSS: ocr_words is a DERIVED index over each document's OCR run, not
|
||||
-- an original record. Dropping it loses no GoBD-relevant data; the word boxes
|
||||
-- are fully regenerable by re-running OCR
|
||||
-- (`archivdms documents reprocess-all`). Only the highlight/overlay feature
|
||||
-- degrades until then.
|
||||
--
|
||||
-- WORM: no archived file in store/ is touched; the FK to documents is only
|
||||
-- consumed here (ON DELETE CASCADE), never the other way round, so dropping
|
||||
-- this table cannot cascade into documents.
|
||||
|
||||
BEGIN;
|
||||
|
||||
DROP INDEX IF EXISTS idx_ocr_words_word_text;
|
||||
DROP INDEX IF EXISTS idx_ocr_words_document;
|
||||
DROP TABLE IF EXISTS ocr_words;
|
||||
|
||||
COMMIT;
|
||||
@@ -0,0 +1,25 @@
|
||||
-- 025_document_date_score.down.sql
|
||||
-- Rollback documentation for 025_document_date_score.sql.
|
||||
-- Documentation only — archivdms has no migration runner; run manually via
|
||||
-- psql after deploying a binary whose initSchema no longer adds the column.
|
||||
--
|
||||
-- PRECONDITION: the ALTER TABLE ... ADD COLUMN IF NOT EXISTS
|
||||
-- document_date_score must be removed from Store.initSchema
|
||||
-- (internal/storage/documents.go) first, and every SELECT/UPDATE naming
|
||||
-- document_date_score (accounting_pull.go, document date endpoint) must be
|
||||
-- gone — otherwise the next start recreates the column and running queries
|
||||
-- fail in between.
|
||||
--
|
||||
-- DATA LOSS: drops the per-document confidence values. documents.document_date
|
||||
-- itself is NOT touched, so no belegdatum is lost — only the quality signal
|
||||
-- the Buchhaltungs-Pull-Filter (score >= 0.75) uses. Recomputable only by
|
||||
-- re-scoring the stored ocr_text.
|
||||
--
|
||||
-- WORM: this is a metadata column on documents; dropping it does not touch
|
||||
-- any archived file in store/ and does not affect retain_until.
|
||||
|
||||
BEGIN;
|
||||
|
||||
ALTER TABLE documents DROP COLUMN IF EXISTS document_date_score;
|
||||
|
||||
COMMIT;
|
||||
@@ -0,0 +1,24 @@
|
||||
-- 026_accounting_api_keys.down.sql
|
||||
-- Rollback documentation for 026_accounting_api_keys.sql.
|
||||
-- Documentation only — archivdms has no migration runner; this file is the
|
||||
-- reviewed, copy-pasteable SQL an operator runs manually (psql) AFTER
|
||||
-- deploying a binary whose initSchema no longer creates the object, and
|
||||
-- inside an explicit transaction.
|
||||
--
|
||||
-- PRECONDITION: internal/storage/accounting_api_keys.go
|
||||
-- (Store.initAccountingAPIKeysSchema) must be removed from storage.New()
|
||||
-- first — otherwise the next process start recreates the table.
|
||||
--
|
||||
-- DATA LOSS: destroys all issued accounting API keys (hashes only, raw keys
|
||||
-- were never stored). Every buchhaltung integration using the Pull-API loses
|
||||
-- access and must be re-issued a new key. No GoBD document data is touched:
|
||||
-- accounting_api_keys holds credentials, not archived records. The audit
|
||||
-- entries referencing "key:<id>" survive in audit_log but their id becomes
|
||||
-- unresolvable — accept this only when the whole feature is being withdrawn.
|
||||
|
||||
BEGIN;
|
||||
|
||||
DROP INDEX IF EXISTS idx_accounting_api_keys_tenant;
|
||||
DROP TABLE IF EXISTS accounting_api_keys;
|
||||
|
||||
COMMIT;
|
||||
@@ -23,6 +23,45 @@ Migrationstool. Sie dient als:
|
||||
3. Der SQL-Inhalt muss exakt dem entsprechen, was `initSchema()` (oder das
|
||||
jeweilige Store-Paket) zur Laufzeit ausführt.
|
||||
4. Migrationen werden nie verändert oder gelöscht, nur ergänzt.
|
||||
5. **Zu jeder neuen Migration gehört eine Down-Datei** `NNN_name.down.sql`
|
||||
(siehe Abschnitt „Rollback-Pfad").
|
||||
|
||||
## Rollback-Pfad (`NNN_name.down.sql`)
|
||||
|
||||
`initSchema()` ist ausschließlich vorwärtsgerichtet — es gibt bewusst keinen
|
||||
automatischen Rollback-Runner (kein Migrationstool, kein Zustandstabellen-
|
||||
Tracking). Für den Ernstfall (fehlerhaftes Release, Rückbau eines Features)
|
||||
braucht es trotzdem ein *reviewtes* Rückbau-SQL. Deshalb gilt ab FDN-02:
|
||||
|
||||
**Jede neue `NNN_name.sql` bekommt eine gleichnamige `NNN_name.down.sql`**
|
||||
mit dem exakten Rückbau der Vorwärts-Migration. Separate Datei statt
|
||||
`-- DOWN`-Abschnitt in derselben Datei, weil Regel 4 („Migrationen werden nie
|
||||
verändert") sonst verletzt würde und weil sich eine Down-Datei fehlerfrei
|
||||
per `psql -f` einspielen lässt, ohne vorher Abschnitte herauszuschneiden.
|
||||
|
||||
Anforderungen an eine Down-Datei:
|
||||
|
||||
1. Header-Kommentar mit Bezug auf die Vorwärts-Migration und der zugehörigen
|
||||
PROJ-/Ticket-Nummer.
|
||||
2. **Precondition** benennen: welche Go-Stelle (`initSchema`, Store-Datei,
|
||||
aufrufende Queries) vorher entfernt bzw. deployt sein muss. Sonst legt der
|
||||
nächste Prozessstart das Objekt sofort wieder an.
|
||||
3. **Datenverlust explizit benennen** — was ist danach unwiederbringlich weg,
|
||||
was ist regenerierbar (z.B. abgeleitete Indizes wie `ocr_words`).
|
||||
4. **WORM/GoBD-Hinweis**: klarstellen, dass kein archiviertes File unter
|
||||
`store/` und kein `retain_until` berührt wird. Down-SQL darf niemals
|
||||
Dokument-Nutzdaten oder Aufbewahrungssperren löschen.
|
||||
5. Idempotent formulieren (`DROP ... IF EXISTS`) und in `BEGIN; ... COMMIT;`
|
||||
klammern.
|
||||
|
||||
Ausführung ist immer **manuell und bewusst** (`psql -f
|
||||
internal/storage/migrations/NNN_name.down.sql`), nie automatisch beim Start.
|
||||
|
||||
Beispiele (rückwirkend ergänzt, dienen als Vorlage):
|
||||
`024_ocr_words.down.sql`, `025_document_date_score.down.sql`,
|
||||
`026_accounting_api_keys.down.sql`. Ältere Migrationen (001–023) haben keine
|
||||
Down-Datei — sie beschreiben das etablierte Kernschema, dessen Rückbau kein
|
||||
realistisches Szenario ist.
|
||||
|
||||
## Vorhandene Migrationen
|
||||
|
||||
|
||||
@@ -340,6 +340,36 @@ func (s *Store) ReapStaleJobs(ctx context.Context, timeout time.Duration, maxRet
|
||||
return len(stale), nil
|
||||
}
|
||||
|
||||
// CountProcessingJobsByStatus liefert die Queue-Länge je Status über ALLE
|
||||
// Mandanten hinweg. Bewusst ohne tenant_id-Filter: einziger Aufrufer ist der
|
||||
// betriebsinterne Prometheus-Endpunkt GET /metrics (FDN-08), der nur
|
||||
// aggregierte Zahlen ohne Mandantenbezug ausgibt — es verlassen keine
|
||||
// mandantenbezogenen Daten das System. Für mandantenbezogene Auswertungen
|
||||
// niemals diese Funktion nutzen.
|
||||
func (s *Store) CountProcessingJobsByStatus(ctx context.Context) (map[string]int64, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT status, count(*) FROM processing_jobs GROUP BY status
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: count processing jobs by status: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := map[string]int64{}
|
||||
for rows.Next() {
|
||||
var status string
|
||||
var n int64
|
||||
if err := rows.Scan(&status, &n); err != nil {
|
||||
return nil, fmt.Errorf("storage: scan processing job count: %w", err)
|
||||
}
|
||||
out[status] = n
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("storage: iterate processing job counts: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// RequeueJob stellt einen Job (typischerweise einen dauerhaft 'failed'
|
||||
// gelaufenen) wieder in die Queue und setzt retry_count zurück. Wird vom
|
||||
// späteren manuellen Retry-Endpunkt (Phase 3) genutzt; tenant-scoped.
|
||||
|
||||
Reference in New Issue
Block a user