package api import ( "context" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "net/http" "os" "path/filepath" "strconv" "strings" "time" "unicode" "archivdms/internal/audit" "archivdms/internal/auth" "archivdms/internal/dateformat" "archivdms/internal/matching" "archivdms/internal/ocr" "archivdms/internal/storage" "archivdms/internal/userstore" ) // taxonomyKinds lists the three entity kinds barcode/matching auto-assignment // runs against on ingest, in lookup order. var taxonomyKinds = []string{"tags", "document_types", "correspondents"} func (s *Server) handleListDocuments(w http.ResponseWriter, r *http.Request) { sess := sessionFromCtx(r.Context()) if sess.TenantID == nil { writeError(w, http.StatusForbidden, "tenant context required") return } // Roles are the outer ACL boundary: domain_admin/superadmin see every // document in the tenant; role 'user' is filtered against // document_visibility via their permission-group memberships. var aclUserID *int64 if !auth.HasRole(sess.Role, userstore.RoleDomainAdmin) { uid := sess.UserID aclUserID = &uid } docs, err := s.store.ListDocuments(r.Context(), *sess.TenantID, aclUserID) if err != nil { writeError(w, http.StatusInternalServerError, "list documents failed") return } writeJSON(w, http.StatusOK, docs) } type createDocumentRequest struct { Title string `json:"title"` DocType string `json:"doc_type"` Correspondent string `json:"correspondent"` StoragePath string `json:"storage_path"` ContentHash string `json:"content_hash"` OCRText string `json:"ocr_text"` } func (s *Server) handleCreateDocument(w http.ResponseWriter, r *http.Request) { sess := sessionFromCtx(r.Context()) if sess.TenantID == nil { writeError(w, http.StatusForbidden, "tenant context required") return } var req createDocumentRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid request body") return } if req.Title == "" || req.StoragePath == "" || req.ContentHash == "" { writeError(w, http.StatusBadRequest, "title, storage_path and content_hash are required") return } doc, err := s.store.CreateDocument(r.Context(), storage.CreateDocumentRequest{ TenantID: *sess.TenantID, Title: req.Title, DocType: req.DocType, Correspondent: req.Correspondent, StoragePath: req.StoragePath, ContentHash: req.ContentHash, OCRText: req.OCRText, CreatedBy: &sess.UserID, }) if err != nil { s.audlog.Log(audit.Entry{EventType: audit.EventDocumentCreate, Username: sess.Username, TenantID: sess.TenantID, Success: false, Detail: err.Error()}) writeError(w, http.StatusInternalServerError, "create document failed") return } s.audlog.Log(audit.Entry{ EventType: audit.EventDocumentCreate, Username: sess.Username, TenantID: sess.TenantID, DocumentID: strconv.FormatInt(doc.ID, 10), Success: true, }) // Full-text search index sync (best-effort, never fails the request). s.store.SyncIndex(r.Context(), doc.ID) writeJSON(w, http.StatusCreated, doc) } func (s *Server) handleGetDocument(w http.ResponseWriter, r *http.Request) { sess := sessionFromCtx(r.Context()) id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) if err != nil || sess.TenantID == nil { writeError(w, http.StatusBadRequest, "invalid document id") return } doc, err := s.store.GetDocument(r.Context(), id, *sess.TenantID) if err != nil { writeError(w, http.StatusNotFound, "document not found") return } writeJSON(w, http.StatusOK, doc) } // handleGetDocumentFile streams the stored file bytes of a document for inline // browser preview (GET /api/documents/{id}/file). It applies the exact same // tenant scoping as handleGetDocument (GetDocument filters WHERE tenant_id) and // returns 404 for both "unknown id" and "wrong tenant" so the endpoint never // reveals whether a document exists outside the caller's tenant. // // Pure read: no audit-log entry, consistent with handleGetDocument (only // writing operations are audited in this project). func (s *Server) handleGetDocumentFile(w http.ResponseWriter, r *http.Request) { sess := sessionFromCtx(r.Context()) id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) if err != nil || sess.TenantID == nil { writeError(w, http.StatusBadRequest, "invalid document id") return } doc, err := s.store.GetDocument(r.Context(), id, *sess.TenantID) if err != nil { writeError(w, http.StatusNotFound, "document not found") return } f, err := os.Open(doc.StoragePath) if err != nil { // WORM store should always hold the file, but never trust the disk. s.logger.Error("document file open failed", "document_id", doc.ID, "tenant_id", *sess.TenantID, "storage_path", doc.StoragePath, "err", err) writeError(w, http.StatusInternalServerError, "file unavailable") return } defer f.Close() ext := filepath.Ext(doc.StoragePath) w.Header().Set("Content-Type", detectMimeType("", ext, doc.StoragePath)) w.Header().Set("Content-Disposition", "inline; filename=\""+safeDownloadName(doc.Title, ext)+"\"") w.Header().Set("X-Content-Type-Options", "nosniff") if _, err := io.Copy(w, f); err != nil { s.logger.Warn("document file stream interrupted", "document_id", doc.ID, "err", err) } } // handleGetDocumentThumbnail serves a small PNG preview of a document // (GET /api/documents/{id}/thumbnail) for the grid/thumbnail list view. // // Thumbnails are lazily generated: on a cache miss the first page (PDF) or the // image itself is rendered to storageCfg.ThumbnailPath()//.png and // then served. Formats without a cheap raster (Office/e-mail) or a failed // render return 404 so the frontend shows a generic file icon instead. Same // tenant scoping as handleGetDocumentFile; pure read, not audited. func (s *Server) handleGetDocumentThumbnail(w http.ResponseWriter, r *http.Request) { sess := sessionFromCtx(r.Context()) id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) if err != nil || sess.TenantID == nil { writeError(w, http.StatusBadRequest, "invalid document id") return } doc, err := s.store.GetDocument(r.Context(), id, *sess.TenantID) if err != nil { writeError(w, http.StatusNotFound, "document not found") return } thumbPath := filepath.Join( s.storageCfg.ThumbnailPath(), strconv.FormatInt(*sess.TenantID, 10), doc.ContentHash+".png", ) // Cache miss: render lazily. A nil generator, unsupported format, or render // failure all fall through to 404 (generic icon in the UI). if _, statErr := os.Stat(thumbPath); statErr != nil { if s.thumbs == nil { writeError(w, http.StatusNotFound, "thumbnail unavailable") return } ext := filepath.Ext(doc.StoragePath) mimeType := detectMimeType("", ext, doc.StoragePath) if err := s.thumbs.Generate(r.Context(), doc.StoragePath, mimeType, thumbPath); err != nil { writeError(w, http.StatusNotFound, "thumbnail unavailable") return } } f, err := os.Open(thumbPath) if err != nil { writeError(w, http.StatusNotFound, "thumbnail unavailable") return } defer f.Close() w.Header().Set("Content-Type", "image/png") w.Header().Set("Cache-Control", "private, max-age=86400") w.Header().Set("X-Content-Type-Options", "nosniff") if _, err := io.Copy(w, f); err != nil { s.logger.Warn("thumbnail stream interrupted", "document_id", doc.ID, "err", err) } } type updateDocumentTitleRequest struct { Title string `json:"title"` } // handleUpdateDocumentTitle renames a document (PATCH /api/documents/{id}). // Title is the only currently editable field via this endpoint — doc_type/ // correspondent already have their own dedicated set-endpoints // (SetDocumentDocType/SetDocumentCorrespondent) since assigning those // re-triggers ACL visibility recomputation. func (s *Server) handleUpdateDocumentTitle(w http.ResponseWriter, r *http.Request) { sess := sessionFromCtx(r.Context()) id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) if err != nil || sess.TenantID == nil { writeError(w, http.StatusBadRequest, "invalid document id") return } var req updateDocumentTitleRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid request body") return } title := strings.TrimSpace(req.Title) if title == "" { writeError(w, http.StatusBadRequest, "title is required") return } if err := s.store.UpdateDocumentTitle(r.Context(), id, *sess.TenantID, title); err != nil { status := http.StatusInternalServerError msg := "update document failed" if errors.Is(err, storage.ErrDocumentNotFound) { status = http.StatusNotFound msg = "document not found" } s.audlog.Log(audit.Entry{EventType: audit.EventDocumentCreate, Username: sess.Username, TenantID: sess.TenantID, DocumentID: r.PathValue("id"), Success: false, Detail: "update_title_failed: " + err.Error()}) writeError(w, status, msg) return } doc, err := s.store.GetDocument(r.Context(), id, *sess.TenantID) if err != nil { writeError(w, http.StatusNotFound, "document not found") return } s.audlog.Log(audit.Entry{EventType: audit.EventDocumentCreate, Username: sess.Username, TenantID: sess.TenantID, DocumentID: r.PathValue("id"), Success: true, Detail: "title_updated"}) writeJSON(w, http.StatusOK, doc) } // setDocumentDateRequest carries an optional belegdatum. DocumentDate is a // pointer so an explicit JSON null clears the field, while an omitted field is // rejected (the endpoint's whole purpose is to set/clear the date). The value, // when present, is an ISO date string "YYYY-MM-DD". type setDocumentDateRequest struct { DocumentDate *string `json:"document_date"` } // handleSetDocumentDate sets or clears a document's belegdatum // (PUT /api/documents/{id}/document-date, body {"document_date":"2024-12-31"|null}). // This is the manual counterpart to the automatic extraction at upload/reprocess // time — the WORM file and its store/// path are NEVER moved, only the // document_date metadata column is updated. Tenant-scoped (s.auth); GetDocument // enforces the WHERE tenant_id ownership check so there is no IDOR hole. func (s *Server) handleSetDocumentDate(w http.ResponseWriter, r *http.Request) { sess := sessionFromCtx(r.Context()) ctx := r.Context() id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) if err != nil || sess.TenantID == nil { writeError(w, http.StatusBadRequest, "invalid document id") return } tenantID := *sess.TenantID docRef := r.PathValue("id") var req setDocumentDateRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid request body") return } var date *time.Time var score *float64 if req.DocumentDate != nil { raw := strings.TrimSpace(*req.DocumentDate) if raw != "" { parsed, perr := time.Parse("2006-01-02", raw) if perr != nil { s.audlog.Log(audit.Entry{EventType: audit.EventDocumentUpdate, Username: sess.Username, TenantID: sess.TenantID, DocumentID: docRef, Success: false, Detail: "set_document_date invalid_format"}) writeError(w, http.StatusBadRequest, "document_date must be an ISO date (YYYY-MM-DD) or null") return } date = &parsed // Manual confirmation always overrides whatever automatic // confidence was previously stored — 1.0 marks "user-confirmed", // never leave a stale heuristic score (0.4-0.9) standing after an // explicit human override. manual := 1.0 score = &manual } } // Ownership check: the document must belong to the caller's tenant. if _, err := s.store.GetDocument(ctx, id, tenantID); err != nil { s.audlog.Log(audit.Entry{EventType: audit.EventDocumentUpdate, Username: sess.Username, TenantID: sess.TenantID, DocumentID: docRef, Success: false, Detail: "set_document_date document_not_found"}) writeError(w, http.StatusNotFound, "document not found") return } if err := s.store.UpdateDocumentDate(ctx, id, tenantID, date, score); err != nil { status := http.StatusInternalServerError msg := "update document_date failed" if errors.Is(err, storage.ErrDocumentNotFound) { status = http.StatusNotFound msg = "document not found" } s.audlog.Log(audit.Entry{EventType: audit.EventDocumentUpdate, Username: sess.Username, TenantID: sess.TenantID, DocumentID: docRef, Success: false, Detail: "set_document_date update_failed: " + err.Error()}) writeError(w, status, msg) return } doc, err := s.store.GetDocument(ctx, id, tenantID) if err != nil { writeError(w, http.StatusNotFound, "document not found") return } detail := "document_date_cleared" if date != nil { detail = "document_date_set:" + date.Format("2006-01-02") } s.audlog.Log(audit.Entry{EventType: audit.EventDocumentUpdate, Username: sess.Username, TenantID: sess.TenantID, DocumentID: docRef, Success: true, Detail: detail}) writeJSON(w, http.StatusOK, doc) } // handleReprocessDocument re-runs the OCR/auto-assignment pipeline on an // already-archived document (POST /api/documents/{id}/reprocess). It exists so // documents whose stored ocr_text was produced by an earlier, buggy OCR run // (e.g. rotated phone photos before the EXIF-orientation fix in // internal/ocr/ocr.go) can be re-extracted without re-uploading the file. // // The WORM file in the store is never touched — only the derived ocr_text // column is refreshed. Runs synchronously in the request, mirroring the // original upload path (no job queue exists yet). Only additive // auto-assignment/workflows run: autoAssignTaxonomy only fills doc_type/ // correspondent when still unset and only attaches (never removes) tags, so a // user's manual classification is preserved. The title is likewise only // re-derived from OCR when the user has never manually renamed the document // (documents.title_manually_set = false), never when the user renamed it. // // Tenant-scoped only (s.auth): a user may reprocess any document visible to // them within their tenant; GetDocument enforces the WHERE tenant_id ACL. func (s *Server) handleReprocessDocument(w http.ResponseWriter, r *http.Request) { sess := sessionFromCtx(r.Context()) ctx := r.Context() id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) if err != nil || sess.TenantID == nil { writeError(w, http.StatusBadRequest, "invalid document id") return } tenantID := *sess.TenantID doc, err := s.ReprocessDocument(ctx, tenantID, id, sess.Username) switch { case err == nil: writeJSON(w, http.StatusOK, doc) case errors.Is(err, ErrReprocessNotFound): writeError(w, http.StatusNotFound, "document not found") case errors.Is(err, ErrReprocessOCRUnavailable): writeError(w, http.StatusServiceUnavailable, "OCR extractor not configured") default: writeError(w, http.StatusInternalServerError, "OCR re-processing failed") } } // ErrReprocessNotFound / ErrReprocessOCRUnavailable are sentinel errors returned // by ReprocessDocument so callers (HTTP handler, CLI bulk job) can map them to // the right status/behaviour without duplicating the pipeline logic. var ( ErrReprocessNotFound = errors.New("reprocess: document not found") ErrReprocessOCRUnavailable = errors.New("reprocess: OCR extractor not configured") ) // ReprocessDocument re-runs the OCR/auto-assignment pipeline on one // already-archived document (the shared core behind POST // /api/documents/{id}/reprocess and the `documents reprocess-all` CLI job). // // It is fully tenant-scoped (GetDocument enforces WHERE tenant_id) and never // touches the WORM file — only derived metadata (ocr_text, title, document_date, // additive taxonomy, on_upload workflows) is refreshed. actor is recorded in the // audit log (a username for HTTP requests, e.g. "cron:reprocess-all" for the CLI). // // Errors are additionally wrapped with the sentinels above for the not-found and // missing-extractor cases; all other failures are returned as-is. Success and // failure are both audited (EventDocumentReprocessed) so the GoBD trail is // complete regardless of caller. // ocrWordsFromResult converts the OCR package's word boxes (already mapped // back into the original file's coordinate space, see // internal/ocr/coords.go) into the storage package's persistence shape. A // nil/empty input yields a nil slice, which storage.ReplaceOCRWords treats as // "delete existing rows, insert none" — the correct behaviour when Result.Words // extraction found nothing or failed best-effort inside Extract. func ocrWordsFromResult(documentID int64, words []ocr.WordBox) []storage.OCRWord { if len(words) == 0 { return nil } out := make([]storage.OCRWord, 0, len(words)) for _, w := range words { out = append(out, storage.OCRWord{ DocumentID: documentID, Page: w.Page, Block: w.Block, Par: w.Par, Line: w.Line, Word: w.Text, Left: w.Left, Top: w.Top, Width: w.Width, Height: w.Height, Confidence: w.Confidence, }) } return out } func (s *Server) ReprocessDocument(ctx context.Context, tenantID, id int64, actor string) (*storage.Document, error) { docIDStr := strconv.FormatInt(id, 10) doc, err := s.store.GetDocument(ctx, id, tenantID) if err != nil { s.audlog.Log(audit.Entry{EventType: audit.EventDocumentReprocessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "document_not_found"}) return nil, fmt.Errorf("%w: %v", ErrReprocessNotFound, err) } if s.ocr == nil { s.audlog.Log(audit.Entry{EventType: audit.EventDocumentReprocessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "ocr_extractor_not_configured"}) return nil, ErrReprocessOCRUnavailable } ext := filepath.Ext(doc.StoragePath) mimeType := detectMimeType("", ext, doc.StoragePath) result, err := s.ocr.Extract(ctx, doc.StoragePath, mimeType) if err != nil { s.logger.Warn("reprocess ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID, "storage_path", doc.StoragePath, "err", err) s.audlog.Log(audit.Entry{EventType: audit.EventDocumentReprocessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "ocr_failed: " + err.Error()}) return nil, fmt.Errorf("reprocess ocr extract: %w", err) } ocrText := result.Text barcodes := result.Barcodes // Replace this document's stored word boxes with the fresh set. Must run // on every reprocess, not just the first OCR pass, so re-OCR (e.g. after // the deskew/OSD fix) never leaves stale word boxes from a prior run // alongside the new ones — best-effort, never fails the reprocess since // ocr_text is already the authoritative persisted result. if err := s.store.ReplaceOCRWords(ctx, id, ocrWordsFromResult(id, result.Words)); err != nil { s.logger.Warn("reprocess replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err) } if err := s.store.UpdateDocumentOCRText(ctx, id, tenantID, ocrText); err != nil { s.audlog.Log(audit.Entry{EventType: audit.EventDocumentReprocessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "update_ocr_text_failed: " + err.Error()}) if errors.Is(err, storage.ErrDocumentNotFound) { return nil, fmt.Errorf("%w: %v", ErrReprocessNotFound, err) } return nil, fmt.Errorf("reprocess update ocr_text: %w", err) } // Refresh the in-memory doc so autoAssignTaxonomy matches against the new // OCR text and the response reflects the updated ocr_text. doc.OCRText = ocrText // Re-derive the title from the improved OCR text, unless the user has // explicitly renamed the document (title_manually_set). This is broader // than the old "Scan DD.MM.YYYY" placeholder check: a title that was // itself auto-derived from a bad, pre-fix OCR run (e.g. "Ent »erVice- // Station" from garbled text) is not a placeholder, but it is still // fair game to improve, since the user never touched it. Best-effort, // failures only logged. if !doc.TitleManuallySet { prefix, dateLayout := s.tenantScanTitleParams(ctx, tenantID) if newTitle := titleFromOCRText(ocrText, prefix, dateLayout); newTitle != "" && newTitle != doc.Title { if err := s.store.UpdateDocumentTitleAuto(ctx, id, tenantID, newTitle); err != nil { s.logger.Warn("reprocess title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err) } else { doc.Title = newTitle } } } // Re-derive the belegdatum (document_date) from the improved OCR text. // Unlike upload, this NEVER moves the WORM file: the store/// path // was fixed at upload time and stays put for the file's whole retention life // — only the metadata column is refreshed. Best-effort, failures only // logged. A newly-nil result (previously-recognised date no longer found) // clears the column, mirroring how the title re-derivation is authoritative. { newDate, newScore, newFound := extractDocumentDateWithScore(ocrText) var datePtr *time.Time var scorePtr *float64 if newFound { datePtr, scorePtr = &newDate, &newScore } if !sameDate(datePtr, doc.DocumentDate) { if err := s.store.UpdateDocumentDate(ctx, id, tenantID, datePtr, scorePtr); err != nil { s.logger.Warn("reprocess document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err) } else { doc.DocumentDate = datePtr doc.DocumentDateScore = scorePtr } } } // Additive auto-assignment (only fills unset doc_type/correspondent, only // attaches tags) — best-effort, never fails the request. if assignWarn := s.autoAssignTaxonomy(ctx, tenantID, doc, barcodes, ocrText); assignWarn != "" { s.logger.Info("reprocess auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn) } // Re-evaluate on_upload workflows — best-effort, never fails the request. if err := s.store.RunWorkflowsForDocument(ctx, tenantID, doc, storage.WorkflowTriggerOnUpload); err != nil { s.logger.Warn("reprocess workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err) } // Re-read so the response reflects any doc_type/correspondent set by // auto-assignment or workflows above. if fresh, err := s.store.GetDocument(ctx, id, tenantID); err == nil { doc = fresh } // Repair/refresh the thumbnail on the same pass as OCR reprocessing. The // WORM file itself never changes on reprocess, so this backfills // has_thumbnail for documents uploaded before eager generation existed, // retries a generator failure from the original upload — and, since // thumbnails are derived (never WORM) artefacts, regenerates unconditionally // so that renderer fixes reach existing documents. Concretely: thumbnails // rendered before -auto-orient was added (see internal/thumbnail) are // baked sideways for EXIF-rotated phone photos and can only be corrected by // overwriting the cached PNG; skipping the render when has_thumbnail was // already true would leave them broken forever. Best-effort, see // generateThumbnailBestEffort. s.generateThumbnailBestEffort(ctx, tenantID, doc.ID, doc.StoragePath, doc.ContentHash, mimeType) s.audlog.Log(audit.Entry{EventType: audit.EventDocumentReprocessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: true, Detail: fmt.Sprintf("ocr_text_len=%d barcodes=%d", len(ocrText), len(barcodes))}) return doc, nil } // setDocTypeRequest / setCorrespondentRequest use pointers so that an explicit // JSON null (or 0) removes the assignment, while an omitted field is rejected. type setDocTypeRequest struct { DocTypeID *int64 `json:"doc_type_id"` } type setCorrespondentRequest struct { CorrespondentID *int64 `json:"correspondent_id"` } // handleSetDocumentDocType assigns (or clears) a document's document type // (PUT /api/documents/{id}/doc-type, body {"doc_type_id": number|null}). // A null/0 value removes the assignment. SetDocumentDocType re-triggers // RecomputeVisibility internally because the type feeds document_type_grants, // so no extra ACL handling is needed here. func (s *Server) handleSetDocumentDocType(w http.ResponseWriter, r *http.Request) { sess := sessionFromCtx(r.Context()) ctx := r.Context() id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) if err != nil || sess.TenantID == nil { writeError(w, http.StatusBadRequest, "invalid document id") return } var req setDocTypeRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid request body") return } var docTypeID int64 if req.DocTypeID != nil { docTypeID = *req.DocTypeID } // Ownership check: the document must belong to the caller's tenant. if _, err := s.store.GetDocument(ctx, id, *sess.TenantID); err != nil { s.audlog.Log(audit.Entry{EventType: audit.EventDocTypeSet, Username: sess.Username, TenantID: sess.TenantID, DocumentID: r.PathValue("id"), Success: false, Detail: "document_not_found"}) writeError(w, http.StatusNotFound, "document not found") return } // Guard against cross-tenant references: the doc_type must belong to the // same tenant (0 means "remove", which needs no lookup). if docTypeID != 0 && !s.taxonomyEntityBelongsToTenant(ctx, "document_types", docTypeID, *sess.TenantID) { s.audlog.Log(audit.Entry{EventType: audit.EventDocTypeSet, Username: sess.Username, TenantID: sess.TenantID, DocumentID: r.PathValue("id"), Success: false, Detail: "invalid_doc_type_id"}) writeError(w, http.StatusBadRequest, "invalid doc_type_id") return } if err := s.store.SetDocumentDocType(ctx, id, *sess.TenantID, docTypeID); err != nil { s.audlog.Log(audit.Entry{EventType: audit.EventDocTypeSet, Username: sess.Username, TenantID: sess.TenantID, DocumentID: r.PathValue("id"), Success: false, Detail: err.Error()}) writeError(w, http.StatusInternalServerError, "set doc_type failed") return } doc, err := s.store.GetDocument(ctx, id, *sess.TenantID) if err != nil { writeError(w, http.StatusNotFound, "document not found") return } s.audlog.Log(audit.Entry{EventType: audit.EventDocTypeSet, Username: sess.Username, TenantID: sess.TenantID, DocumentID: r.PathValue("id"), Success: true, Detail: fmt.Sprintf("doc_type_id=%d", docTypeID)}) writeJSON(w, http.StatusOK, doc) } // handleSetDocumentCorrespondent assigns (or clears) a document's correspondent // (PUT /api/documents/{id}/correspondent, body {"correspondent_id": number|null}). // A null/0 value removes the assignment. The correspondent is not part of the // ACL, so SetDocumentCorrespondent only re-syncs the search index. func (s *Server) handleSetDocumentCorrespondent(w http.ResponseWriter, r *http.Request) { sess := sessionFromCtx(r.Context()) ctx := r.Context() id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) if err != nil || sess.TenantID == nil { writeError(w, http.StatusBadRequest, "invalid document id") return } var req setCorrespondentRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid request body") return } var correspondentID int64 if req.CorrespondentID != nil { correspondentID = *req.CorrespondentID } if _, err := s.store.GetDocument(ctx, id, *sess.TenantID); err != nil { s.audlog.Log(audit.Entry{EventType: audit.EventCorrespondentSet, Username: sess.Username, TenantID: sess.TenantID, DocumentID: r.PathValue("id"), Success: false, Detail: "document_not_found"}) writeError(w, http.StatusNotFound, "document not found") return } if correspondentID != 0 && !s.taxonomyEntityBelongsToTenant(ctx, "correspondents", correspondentID, *sess.TenantID) { s.audlog.Log(audit.Entry{EventType: audit.EventCorrespondentSet, Username: sess.Username, TenantID: sess.TenantID, DocumentID: r.PathValue("id"), Success: false, Detail: "invalid_correspondent_id"}) writeError(w, http.StatusBadRequest, "invalid correspondent_id") return } if err := s.store.SetDocumentCorrespondent(ctx, id, *sess.TenantID, correspondentID); err != nil { s.audlog.Log(audit.Entry{EventType: audit.EventCorrespondentSet, Username: sess.Username, TenantID: sess.TenantID, DocumentID: r.PathValue("id"), Success: false, Detail: err.Error()}) writeError(w, http.StatusInternalServerError, "set correspondent failed") return } doc, err := s.store.GetDocument(ctx, id, *sess.TenantID) if err != nil { writeError(w, http.StatusNotFound, "document not found") return } s.audlog.Log(audit.Entry{EventType: audit.EventCorrespondentSet, Username: sess.Username, TenantID: sess.TenantID, DocumentID: r.PathValue("id"), Success: true, Detail: fmt.Sprintf("correspondent_id=%d", correspondentID)}) writeJSON(w, http.StatusOK, doc) } // taxonomyEntityBelongsToTenant reports whether a taxonomy entity id exists // within the tenant's scope (prevents cross-tenant IDOR assignment). Uses the // tenant-scoped list rather than a raw id lookup because the store has no // GetTaxonomyEntity-by-id method. func (s *Server) taxonomyEntityBelongsToTenant(ctx context.Context, kind string, id, tenantID int64) bool { entities, err := s.store.ListTaxonomyEntities(ctx, kind, tenantID) if err != nil { return false } for _, e := range entities { if e.ID == id { return true } } return false } // handleDeleteDocument moves a document into the trash (soft-delete). The WORM // file stays physically untouched until a two-person-confirmed delete request // is executed via the trash endpoints (internal/api/trash_handlers.go). This // replaces the former hard DELETE to satisfy the GoBD staged-deletion concept. func (s *Server) handleDeleteDocument(w http.ResponseWriter, r *http.Request) { sess := sessionFromCtx(r.Context()) id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) if err != nil || sess.TenantID == nil { writeError(w, http.StatusBadRequest, "invalid document id") return } if err := s.store.SoftDeleteDocument(r.Context(), id, *sess.TenantID, sess.UserID); err != nil { status := http.StatusBadRequest msg := "delete document failed" if errors.Is(err, storage.ErrDocumentNotInTrash) { status = http.StatusNotFound msg = "document not found" } else if errors.Is(err, storage.ErrAlreadyInTrash) { status = http.StatusConflict msg = "document already in trash" } s.audlog.Log(audit.Entry{EventType: audit.EventDocumentTrash, Username: sess.Username, TenantID: sess.TenantID, DocumentID: r.PathValue("id"), Success: false, Detail: err.Error()}) writeError(w, status, msg) return } s.audlog.Log(audit.Entry{EventType: audit.EventDocumentTrash, Username: sess.Username, TenantID: sess.TenantID, DocumentID: r.PathValue("id"), Success: true}) writeJSON(w, http.StatusOK, map[string]string{"status": "trashed"}) } // generateThumbnailBestEffort renders and caches a preview thumbnail for a // just-(re)processed document, then records the result via // SetDocumentHasThumbnail. Shared by storeUploadedFile (eager generation at // upload) and ReprocessDocument (repairs a thumbnail that failed/was skipped // at upload time — e.g. the generator wasn't configured yet, or a transient // convert/pdftoppm failure). Mirrors the cache path/layout used by the lazy // GET /api/documents/{id}/thumbnail handler exactly, so whichever path wins // the race, the other sees a warm cache. Never returns an error — any // failure (nil generator, unsupported mime, subprocess failure) is only // logged at Warn, matching the "OCR/thumbnail never blocks the request" // convention used throughout this file. func (s *Server) generateThumbnailBestEffort(ctx context.Context, tenantID, documentID int64, storagePath, contentHash, mimeType string) { if s.thumbs == nil { return } thumbPath := filepath.Join(s.storageCfg.ThumbnailPath(), strconv.FormatInt(tenantID, 10), contentHash+".png") if err := s.thumbs.Generate(ctx, storagePath, mimeType, thumbPath); err != nil { s.logger.Warn("eager thumbnail generation failed", "document_id", documentID, "tenant_id", tenantID, "mime_type", mimeType, "err", err) return } if err := s.store.SetDocumentHasThumbnail(ctx, documentID, tenantID, true); err != nil { s.logger.Warn("has_thumbnail flag update failed", "document_id", documentID, "tenant_id", tenantID, "err", err) } } // --- upload pipeline (multipart -> inbox -> hash -> store -> OCR -> DB) --- // handleUploadDocument accepts a real multipart file upload (unlike // handleCreateDocument, which only records metadata a caller already // computed elsewhere). It writes the raw upload to the tenant's inbox // directory, hashes it (SHA-256), moves it into the content-addressed WORM // store (store////.), locks it down with // chmod 0440, runs best-effort OCR, and finally records the document in the // database. See dms-featureliste-prompt.md / lazy-splashing-puppy plan for // the full storage layout rationale. func (s *Server) handleUploadDocument(w http.ResponseWriter, r *http.Request) { sess := sessionFromCtx(r.Context()) if sess.TenantID == nil { writeError(w, http.StatusForbidden, "tenant context required") return } tenantID := *sess.TenantID maxBytes := int64(s.storageCfg.ResolvedMaxUploadSizeMB()) * 1024 * 1024 s.logger.Info("upload request received", "username", sess.Username, "tenant_id", tenantID, "content_length", r.ContentLength, "max_bytes", maxBytes, "remote_addr", r.RemoteAddr) r.Body = http.MaxBytesReader(w, r.Body, maxBytes) if err := r.ParseMultipartForm(32 << 20); err != nil { // Logged explicitly (not just audit) because this is the most likely // point for a silent client-side abort or an oversized upload // hitting MaxBytesReader — both close the connection before any // later log/audit call would otherwise run, which previously left // zero trace of the failure in the backend logs (see DEVLOG 2026-07-15). s.logger.Warn("upload parse failed", "username", sess.Username, "tenant_id", tenantID, "content_length", r.ContentLength, "max_bytes", maxBytes, "err", err) s.audlog.Log(audit.Entry{ EventType: audit.EventDocumentCreate, Username: sess.Username, TenantID: sess.TenantID, Success: false, Detail: fmt.Sprintf("upload parse failed: %v (content_length=%d max_bytes=%d)", err, r.ContentLength, maxBytes), }) writeError(w, http.StatusBadRequest, "invalid or oversized multipart request") return } // title is optional: if left blank (e.g. the /scan quick-capture flow), // storeUploadedFile derives one from the OCR text once extraction has // run, falling back to a timestamp placeholder if OCR yields nothing. title := strings.TrimSpace(r.FormValue("title")) docType := strings.TrimSpace(r.FormValue("doc_type")) correspondent := strings.TrimSpace(r.FormValue("correspondent")) file, header, err := r.FormFile("file") if err != nil { writeError(w, http.StatusBadRequest, "file field is required") return } defer file.Close() uploaderID := sess.UserID doc, warn, err := s.storeUploadedFile(r.Context(), tenantID, title, docType, correspondent, file, header.Filename, header.Header.Get("Content-Type"), &uploaderID) if err != nil { if errors.Is(err, storage.ErrDuplicateContentHash) { s.audlog.Log(audit.Entry{EventType: audit.EventDocumentCreate, Username: sess.Username, TenantID: sess.TenantID, Success: false, Detail: "duplicate content hash"}) writeError(w, http.StatusConflict, "a document with identical content already exists") return } s.audlog.Log(audit.Entry{EventType: audit.EventDocumentCreate, Username: sess.Username, TenantID: sess.TenantID, Success: false, Detail: err.Error()}) writeError(w, http.StatusInternalServerError, "upload failed") return } entry := audit.Entry{ EventType: audit.EventDocumentCreate, Username: sess.Username, TenantID: sess.TenantID, DocumentID: strconv.FormatInt(doc.ID, 10), Success: true, } if warn != "" { entry.Detail = warn } s.audlog.Log(entry) writeJSON(w, http.StatusCreated, doc) } // StoreUploadedFile is the exported wrapper around storeUploadedFile, used // by internal/sftpserver's watcher (via the sftpserver.UploadFunc callback // wired up in cmd/archivdms/main.go) so both the HTTP upload endpoint and // the SFTP watcher run through the exact same pipeline. func (s *Server) StoreUploadedFile( ctx context.Context, tenantID int64, title, docType, correspondent string, file io.Reader, filename, contentType string, ) (*storage.Document, string, error) { // No interactive session for the SFTP watcher, so no uploader to attribute // ownership-based visibility to — the document falls back to the normal // grant-based ACL like before. return s.storeUploadedFile(ctx, tenantID, title, docType, correspondent, file, filename, contentType, nil) } // storeUploadedFile is the SYNCHRONOUS half of the upload pipeline (the // "stageDocument" step of the queue split, see internal/jobqueue): inbox -> // hash -> duplicate check -> WORM move + chmod 0440 -> atomic // document+job INSERT. Everything expensive and derivable (OCR extraction, // taxonomy auto-assignment, on_upload workflows) is NOT done here anymore — // it runs asynchronously in ProcessDocumentJob, dispatched from the // per-tenant job queue. That keeps batch scans / SFTP mass uploads from // blocking request goroutines behind Tesseract. // // What deliberately stays synchronous: // - the WORM guarantee (hash, content-addressed store path, chmod 0440), // so a request that returns 201 has the file archived immutably; // - the duplicate detection (filesystem + DB unique index); // - the eager thumbnail render (cheap-ish, purely read-only on the WORM // file, and the UI wants it immediately). // // Because OCR no longer runs before the move, the store/// path is // now derived from the upload time instead of the recognised belegdatum. The // belegdatum is still extracted later by the job and written to // documents.document_date — but the archived file is never moved afterwards // (WORM), exactly like ReprocessDocument has always behaved. // // The returned warning string is kept for API compatibility with the previous // synchronous implementation; it is now only used for staging-level notes. // // The signature intentionally takes a plain io.Reader + filename/contentType // instead of *multipart.FileHeader so it can be shared between the HTTP // upload handler and the SFTP watcher (internal/sftpserver), which has no // multipart request to draw a header from. func (s *Server) storeUploadedFile( ctx context.Context, tenantID int64, title, docType, correspondent string, file io.Reader, filename, contentType string, createdBy *int64, ) (*storage.Document, string, error) { ext := filepath.Ext(filename) // 1. Write to inbox//., hashing as we go. inboxDir := filepath.Join(s.storageCfg.InboxPath(), strconv.FormatInt(tenantID, 10)) if err := os.MkdirAll(inboxDir, 0o750); err != nil { return nil, "", fmt.Errorf("create inbox dir: %w", err) } inboxName := randomUploadID() + ext inboxPath := filepath.Join(inboxDir, inboxName) inboxFile, err := os.OpenFile(inboxPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o640) if err != nil { return nil, "", fmt.Errorf("create inbox file: %w", err) } hasher := sha256.New() if _, err := io.Copy(io.MultiWriter(inboxFile, hasher), file); err != nil { inboxFile.Close() os.Remove(inboxPath) return nil, "", fmt.Errorf("write inbox file: %w", err) } if err := inboxFile.Close(); err != nil { os.Remove(inboxPath) return nil, "", fmt.Errorf("close inbox file: %w", err) } contentHash := hex.EncodeToString(hasher.Sum(nil)) // 1a. Trennseiten-Split (internal/pagesplit): if this is a multi-page PDF // scan stack with barcode separator sheets in it, cut it into one document // per segment BEFORE anything is archived — the separator sheets are // control pages, not content, and the individual receipts belong in the // archive as individual documents. Runs here, on the inbox scratch file, // precisely because after the WORM move nothing may be re-derived from the // file any more. Disabled by default and fail-safe: any problem falls // through to the normal single-document path below. if doc, warn, handled, splitErr := s.trySplitStagedUpload(ctx, tenantID, inboxPath, ext, contentHash, title, docType, correspondent, filename, contentType, createdBy); handled { return doc, warn, splitErr } return s.archiveStagedFile(ctx, tenantID, inboxPath, ext, contentHash, title, docType, correspondent, contentType, createdBy) } // archiveStagedFile is the second half of the synchronous staging path, // factored out of storeUploadedFile so the separator-page split // (trySplitStagedUpload) can run each of its part PDFs through the exact same // archival steps as a plain single-file upload: duplicate pre-check, WORM move // + chmod 0440, atomic document+job INSERT, eager thumbnail, index sync. // // srcPath is a scratch file the caller no longer owns afterwards — on success // it has been MOVED into the WORM store (or copied+removed on a cross-device // fallback), on failure it is removed. contentHash must be the SHA-256 of // srcPath's current contents; the caller computes it while writing (upload) or // via hashFile (split parts). // // Fully tenant-scoped: the store path embeds tenantID and every store call // filters on it. func (s *Server) archiveStagedFile( ctx context.Context, tenantID int64, inboxPath, ext, contentHash string, title, docType, correspondent, contentType string, createdBy *int64, ) (*storage.Document, string, error) { // 1b. Early duplicate check — BEFORE OCR. If this tenant already has a // document with the same content_hash, the file is a byte-identical // re-upload: skip the expensive Tesseract/poppler OCR run entirely, drop // the inbox scratch file, and reject as duplicate. This saves OCR load and // upholds GoBD single-storage. The filesystem collision check (step 5) and // the DB unique index remain as the second/third line of defence against a // race between the check here and the INSERT below. if exists, err := s.store.DocumentExistsByHash(ctx, tenantID, contentHash); err != nil { os.Remove(inboxPath) return nil, "", fmt.Errorf("duplicate pre-check: %w", err) } else if exists { os.Remove(inboxPath) return nil, "", storage.ErrDuplicateContentHash } // 2./3. WORM target directory's year/month. Since OCR moved into the async // job, the belegdatum is not known yet at this point — the archival folder // therefore follows the scan/upload time. documents.document_date is filled // in later by ProcessDocumentJob; the already-archived file is never moved // afterwards (WORM), same as on reprocess. var warn string if s.ocr == nil { warn = "OCR extractor not configured" } pathDate := time.Now() // 4. Build the WORM target path store////.. storeDir := filepath.Join(s.storageCfg.StorePath(), strconv.FormatInt(tenantID, 10), fmt.Sprintf("%04d", pathDate.Year()), fmt.Sprintf("%02d", pathDate.Month())) if err := os.MkdirAll(storeDir, 0o750); err != nil { os.Remove(inboxPath) return nil, "", fmt.Errorf("create store dir: %w", err) } storePath := filepath.Join(storeDir, contentHash+ext) // 5. Collision check: identical hash already stored -> reject as // duplicate before touching the DB (filesystem-level half of the // duplicate protection; the DB unique index is the other half). if _, err := os.Stat(storePath); err == nil { os.Remove(inboxPath) return nil, "", storage.ErrDuplicateContentHash } else if !os.IsNotExist(err) { os.Remove(inboxPath) return nil, "", fmt.Errorf("stat store path: %w", err) } // 6. Move inbox -> store. Prefer atomic rename; fall back to copy+remove // if inbox/store ever end up on different filesystems/mounts. if err := os.Rename(inboxPath, storePath); err != nil { if copyErr := copyFile(inboxPath, storePath); copyErr != nil { os.Remove(inboxPath) return nil, "", fmt.Errorf("move file to store: rename failed (%v), copy fallback failed: %w", err, copyErr) } os.Remove(inboxPath) } // 7. WORM lock: read-only, no write access for anyone once archived. This is // the ONLY chmod and it happens exactly once, after the file reaches its // final path — the file is never moved or renamed again afterwards. if err := os.Chmod(storePath, 0o440); err != nil { return nil, "", fmt.Errorf("chmod store file: %w", err) } // 8. Record the document AND its processing job in ONE transaction. If no // title was supplied we can only put a timestamp placeholder here (OCR has // not run yet) and flag the job with deriveTitle=true so the worker // replaces it with a real, OCR-derived title. A caller-supplied title // (HTTP form field, SFTP filename) is never overwritten — mirroring the // previous synchronous behaviour, where a non-empty title skipped the OCR // derivation entirely. deriveTitle := title == "" if deriveTitle { prefix, dateLayout := s.tenantScanTitleParams(ctx, tenantID) title = titleFromOCRText("", prefix, dateLayout) } doc, job, err := s.store.CreateDocumentWithJob(ctx, storage.CreateDocumentRequest{ TenantID: tenantID, Title: title, DocType: docType, Correspondent: correspondent, StoragePath: storePath, ContentHash: contentHash, Source: "upload", CreatedBy: createdBy, }, deriveTitle) if err != nil { return nil, "", err } s.logger.Info("document staged, processing job queued", "document_id", doc.ID, "tenant_id", tenantID, "job_id", job.ID, "derive_title", deriveTitle) // 7b. Eager thumbnail generation: same pipeline/cache path as the lazy // on-demand render in handleGetDocumentThumbnail // (storageCfg.ThumbnailPath()//.png), just run once now // instead of on first view. Reads the already-archived WORM file // read-only (never touches/rewrites store/). Best-effort only — a missing // generator, unsupported format (Office/e-mail), or a convert/pdftoppm // failure must never fail the upload; has_thumbnail simply stays false and // the thumbnail endpoint's existing lazy-generation fallback still // applies on first view. s.generateThumbnailBestEffort(ctx, tenantID, doc.ID, storePath, contentHash, detectMimeType(contentType, ext, storePath)) // 9. Initial full-text index sync so the document is findable by title even // before its OCR text exists. ProcessDocumentJob syncs again once the OCR // text and taxonomy are in place. Best-effort, never fails the upload. s.store.SyncIndex(ctx, doc.ID) // OCR, taxonomy auto-assignment and on_upload workflows now happen // asynchronously in ProcessDocumentJob (dispatched by internal/jobqueue). return doc, warn, nil } // splitActor is the audit-log actor for ingest-time separator splitting. Like // "jobqueue" in ProcessDocumentJob, this is a pipeline step rather than a user // action, and it must produce the same entry whether the upload came from the // HTTP endpoint or the SFTP watcher (which has no session at all). const splitActor = "ingest" // trySplitStagedUpload implements the barcode separator-page split // ("Trennseiten-Split") for a freshly staged, not yet archived upload. // // handled == true means this function has taken full responsibility for the // upload: the inbox file is gone, N part documents have been archived and // queued, and the caller must return (doc, warn, err) unchanged instead of // running the normal single-document path. handled == false means "not // applicable, carry on normally" and leaves inboxPath untouched — that is the // outcome for a disabled detector, a non-PDF upload, a document without // separator pages, and every detection/split failure. A scan stack that cannot // be analysed is always archived unsplit rather than rejected. // // GoBD-Nachvollziehbarkeit: the uploaded original is deliberately NOT archived // when a split happens (archiving both the stack and its parts would duplicate // every page and break the single-storage principle), so the audit entry is // the sole record connecting the parts back to the upload. It therefore names // the original filename, its SHA-256, the page count, the dropped separator // pages, and the page range + document ID of every part. Failures are logged // with Success:false as well, so "split was attempted and abandoned" is // distinguishable from "split never ran". func (s *Server) trySplitStagedUpload( ctx context.Context, tenantID int64, inboxPath, ext, contentHash string, title, docType, correspondent, filename, contentType string, createdBy *int64, ) (*storage.Document, string, bool, error) { if s.pagesplitter == nil || !s.pagesplitter.Enabled { return nil, "", false, nil } if detectMimeType(contentType, ext, inboxPath) != "application/pdf" { return nil, "", false, nil } res, split, err := s.pagesplitter.Split(ctx, inboxPath) if err != nil { s.logger.Warn("separator-page split failed, archiving document unsplit", "tenant_id", tenantID, "filename", filename, "err", err) s.audlog.Log(audit.Entry{ EventType: audit.EventDocumentSplit, Username: splitActor, TenantID: &tenantID, Success: false, Detail: fmt.Sprintf("split aborted, archived unsplit: file=%q hash=%s err=%v", filename, contentHash, err), }) return nil, "", false, nil } if !split || res == nil || len(res.Parts) == 0 { return nil, "", false, nil } defer res.Cleanup() // The stack itself never becomes an archive object — its pages live on in // the parts. Removing it here (before the parts are written) also keeps the // inbox free of a scratch file if part archival fails halfway. os.Remove(inboxPath) var ( docs []*storage.Document partIDs []string firstErr error dupCount int partNotes []string ) for i, partPath := range res.Parts { partHash, herr := hashFile(partPath) if herr != nil { if firstErr == nil { firstErr = fmt.Errorf("hash split part %d: %w", i+1, herr) } partNotes = append(partNotes, fmt.Sprintf("part %d: hash failed (%v)", i+1, herr)) continue } // An explicitly supplied title applies to the whole stack, so each part // gets it suffixed to stay distinguishable. An empty title is passed // through unchanged so archiveStagedFile flags the job with // deriveTitle=true and the part gets its own OCR-derived title. partTitle := title if partTitle != "" { partTitle = fmt.Sprintf("%s (Teil %d)", partTitle, i+1) } doc, _, aerr := s.archiveStagedFile(ctx, tenantID, partPath, ".pdf", partHash, partTitle, docType, correspondent, "application/pdf", createdBy) if aerr != nil { if errors.Is(aerr, storage.ErrDuplicateContentHash) { dupCount++ partNotes = append(partNotes, fmt.Sprintf("part %d: duplicate, skipped", i+1)) } else { if firstErr == nil { firstErr = aerr } partNotes = append(partNotes, fmt.Sprintf("part %d: %v", i+1, aerr)) } continue } docs = append(docs, doc) partIDs = append(partIDs, fmt.Sprintf("%d(p%d-%d)", doc.ID, res.PartPageRanges[i][0], res.PartPageRanges[i][1])) } detail := fmt.Sprintf("file=%q hash=%s pages=%d separator_pages=%v parts=%d documents=%s", filename, contentHash, res.PageCount, res.SeparatorPages, len(res.Parts), strings.Join(partIDs, ",")) if len(partNotes) > 0 { detail += " notes=" + strings.Join(partNotes, "; ") } if len(docs) == 0 { s.audlog.Log(audit.Entry{ EventType: audit.EventDocumentSplit, Username: splitActor, TenantID: &tenantID, Success: false, Detail: detail, }) if firstErr != nil { return nil, "", true, firstErr } // Every part already existed byte-identically: the whole stack is a // re-scan, reported to the caller as the usual duplicate conflict. return nil, "", true, storage.ErrDuplicateContentHash } s.audlog.Log(audit.Entry{ EventType: audit.EventDocumentSplit, Username: splitActor, TenantID: &tenantID, DocumentID: strconv.FormatInt(docs[0].ID, 10), Success: true, Detail: detail, }) s.logger.Info("upload split at barcode separator pages", "tenant_id", tenantID, "filename", filename, "pages", res.PageCount, "separator_pages", res.SeparatorPages, "parts", len(res.Parts), "documents_created", len(docs), "duplicates_skipped", dupCount) warn := fmt.Sprintf("Trennseiten-Split: %d Teildokumente aus %d Seiten erzeugt", len(docs), res.PageCount) if dupCount > 0 || firstErr != nil { warn += "; " + strings.Join(partNotes, "; ") } // The HTTP handler returns exactly one document; the first part is the // natural choice (the UI reloads the list afterwards anyway). All part IDs // are in the audit entry above. return docs[0], warn, true, nil } // hashFile computes the SHA-256 of a file on disk. Used for split parts, which // — unlike a streamed upload — are produced by poppler on disk and therefore // cannot be hashed while being written. func hashFile(path string) (string, error) { f, err := os.Open(path) if err != nil { return "", err } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil } // ProcessDocumentJob is the ASYNCHRONOUS half of the upload pipeline: it runs // OCR extraction, derives title/belegdatum, performs taxonomy // auto-assignment and executes the on_upload workflows for one already-staged, // already-archived document. It is the callback the job-queue dispatcher // (internal/jobqueue) invokes for every claimed processing_jobs row — wired in // cmd/archivdms/main.go, the same "function value instead of import" // arrangement used for sftpserver.UploadFunc, to keep internal/jobqueue free // of an internal/api dependency. // // It NEVER touches the WORM file: the archived blob is only ever read; all // results land in derived metadata columns. Fully tenant-scoped (every store // call filters on tenant_id). // // Returning an error tells the dispatcher the job failed, which triggers the // retry/backoff bookkeeping in storage.MarkJobFailed. Only genuinely fatal // problems (document gone, OCR extractor missing/failing) return an error; // best-effort steps (title, date, taxonomy, workflows) are logged and do not // fail the job, since their input (the OCR text) is already persisted. // // deriveTitle mirrors processing_jobs.derive_title: true only when the upload // carried no explicit title and the placeholder may be replaced. func (s *Server) ProcessDocumentJob(ctx context.Context, tenantID, documentID int64, deriveTitle bool) error { docIDStr := strconv.FormatInt(documentID, 10) actor := "jobqueue" doc, err := s.store.GetDocument(ctx, documentID, tenantID) if err != nil { s.audlog.Log(audit.Entry{EventType: audit.EventDocumentProcessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "document_not_found"}) return fmt.Errorf("%w: %v", ErrReprocessNotFound, err) } if s.ocr == nil { s.audlog.Log(audit.Entry{EventType: audit.EventDocumentProcessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "ocr_extractor_not_configured"}) return ErrReprocessOCRUnavailable } ext := filepath.Ext(doc.StoragePath) mimeType := detectMimeType("", ext, doc.StoragePath) result, err := s.ocr.Extract(ctx, doc.StoragePath, mimeType) if err != nil { s.logger.Warn("jobqueue ocr extraction failed", "document_id", doc.ID, "tenant_id", tenantID, "storage_path", doc.StoragePath, "resolved_mime", mimeType, "err", err) s.audlog.Log(audit.Entry{EventType: audit.EventDocumentProcessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "ocr_failed: " + err.Error()}) return fmt.Errorf("jobqueue ocr extract: %w", err) } ocrText := result.Text barcodes := result.Barcodes // Persist word-level bounding boxes for this OCR pass. ReplaceOCRWords // clears any prior rows first so a job retry (see storage.MarkJobFailed // retry/backoff) never accumulates duplicates. Best-effort, never fails // the job since ocr_text is already the authoritative persisted result. if err := s.store.ReplaceOCRWords(ctx, documentID, ocrWordsFromResult(documentID, result.Words)); err != nil { s.logger.Warn("jobqueue replace ocr_words failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err) } if err := s.store.UpdateDocumentOCRText(ctx, documentID, tenantID, ocrText); err != nil { s.audlog.Log(audit.Entry{EventType: audit.EventDocumentProcessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: false, Detail: "update_ocr_text_failed: " + err.Error()}) return fmt.Errorf("jobqueue update ocr_text: %w", err) } doc.OCRText = ocrText // Replace the staging placeholder title with a real one derived from the // OCR text — only when the upload had no explicit title (deriveTitle) and // the user has not renamed the document in the meantime. if deriveTitle && !doc.TitleManuallySet { prefix, dateLayout := s.tenantScanTitleParams(ctx, tenantID) if newTitle := titleFromOCRText(ocrText, prefix, dateLayout); newTitle != "" && newTitle != doc.Title { if err := s.store.UpdateDocumentTitleAuto(ctx, documentID, tenantID, newTitle); err != nil { s.logger.Warn("jobqueue title update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err) } else { doc.Title = newTitle } } } // Belegdatum from the OCR text. The store/// path was already // fixed at staging time (upload date) and is NEVER moved — only the // metadata column is written here. { newDate, newScore, newFound := extractDocumentDateWithScore(ocrText) var datePtr *time.Time var scorePtr *float64 if newFound { datePtr, scorePtr = &newDate, &newScore } if !sameDate(datePtr, doc.DocumentDate) { if err := s.store.UpdateDocumentDate(ctx, documentID, tenantID, datePtr, scorePtr); err != nil { s.logger.Warn("jobqueue document_date update failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err) } else { doc.DocumentDate = datePtr doc.DocumentDateScore = scorePtr } } } if assignWarn := s.autoAssignTaxonomy(ctx, tenantID, doc, barcodes, ocrText); assignWarn != "" { s.logger.Info("jobqueue auto-assignment", "document_id", doc.ID, "tenant_id", tenantID, "detail", assignWarn) } if err := s.store.RunWorkflowsForDocument(ctx, tenantID, doc, storage.WorkflowTriggerOnUpload); err != nil { s.logger.Warn("jobqueue workflow execution failed", "document_id", doc.ID, "tenant_id", tenantID, "err", err) } // Final index sync: auto-assignment/workflows may have re-indexed already, // but a document without any assignment still needs its OCR text indexed. s.store.SyncIndex(ctx, documentID) s.audlog.Log(audit.Entry{EventType: audit.EventDocumentProcessed, Username: actor, TenantID: &tenantID, DocumentID: docIDStr, Success: true, Detail: fmt.Sprintf("ocr_text_len=%d barcodes=%d derive_title=%t", len(ocrText), len(barcodes), deriveTitle)}) return nil } // autoAssignTaxonomy runs the barcode-value lookup and the matching engine // (internal/matching) against all active (match_algorithm != 'none') // tags/document_types/correspondents of the tenant, assigning hits to doc. // Returns a short warning string (empty if nothing noteworthy happened) to // be folded into the caller's audit-log warning/detail. func (s *Server) autoAssignTaxonomy(ctx context.Context, tenantID int64, doc *storage.Document, barcodes []string, ocrText string) string { var notes []string // Persist the raw barcode payloads regardless of match, for GoBD // Nachvollziehbarkeit even when nothing matched. if len(barcodes) > 0 { if err := s.store.SetDocumentBarcodeValues(ctx, doc.ID, tenantID, barcodes); err != nil { s.logger.Warn("failed to store barcode values", "document_id", doc.ID, "err", err) } } assignedTags := map[int64]bool{} var assignedDocType, assignedCorrespondent int64 // 1. Barcode-value lookup against all three kinds. for _, code := range barcodes { for _, kind := range taxonomyKinds { entity, err := s.store.GetTaxonomyEntityByBarcode(ctx, kind, tenantID, code) if err != nil { continue // no match for this (kind, code) pair, not an error } switch kind { case "tags": if !assignedTags[entity.ID] { if err := s.store.AttachTag(ctx, doc.ID, entity.ID); err == nil { assignedTags[entity.ID] = true notes = append(notes, "tag:"+entity.Name+" (barcode)") } } case "document_types": if assignedDocType == 0 { if err := s.store.SetDocumentDocType(ctx, doc.ID, tenantID, entity.ID); err == nil { assignedDocType = entity.ID notes = append(notes, "doc_type:"+entity.Name+" (barcode)") } } case "correspondents": if assignedCorrespondent == 0 { if err := s.store.SetDocumentCorrespondent(ctx, doc.ID, tenantID, entity.ID); err == nil { assignedCorrespondent = entity.ID notes = append(notes, "correspondent:"+entity.Name+" (barcode)") } } } } } // 2. Matching engine against ocr_text + title, for all active matchers. haystack := doc.Title if ocrText != "" { haystack = doc.Title + "\n" + ocrText } for _, kind := range taxonomyKinds { entities, err := s.store.ListActiveMatchers(ctx, kind, tenantID) if err != nil { s.logger.Warn("failed to list active matchers", "kind", kind, "err", err) continue } for _, entity := range entities { if !matching.Match(entity.MatchAlgorithm, entity.MatchPattern, entity.CaseSensitive, haystack) { continue } switch kind { case "tags": if !assignedTags[entity.ID] { if err := s.store.AttachTag(ctx, doc.ID, entity.ID); err == nil { assignedTags[entity.ID] = true notes = append(notes, "tag:"+entity.Name+" (matched)") } } case "document_types": if assignedDocType == 0 { if err := s.store.SetDocumentDocType(ctx, doc.ID, tenantID, entity.ID); err == nil { assignedDocType = entity.ID notes = append(notes, "doc_type:"+entity.Name+" (matched)") } } case "correspondents": if assignedCorrespondent == 0 { if err := s.store.SetDocumentCorrespondent(ctx, doc.ID, tenantID, entity.ID); err == nil { assignedCorrespondent = entity.ID notes = append(notes, "correspondent:"+entity.Name+" (matched)") } } } } } if len(notes) == 0 { return "" } return "auto-assigned: " + strings.Join(notes, ", ") } // maxDerivedTitleLen caps a title auto-derived from OCR text so an // unusually long first line doesn't produce an unwieldy document title. const maxDerivedTitleLen = 120 // exampleScanTitleFormats are non-binding suggestions surfaced to the frontend // as clickable starting points. They are NOT a validation constraint — admins // may enter any free token pattern (see dateformat.Translate). Keep the first // entry equal to defaultScanTitleDateFormat for a sensible default suggestion. var exampleScanTitleFormats = []string{ "DD.MM.YYYY HH:mm", // German (default) "YYYY-MM-DD HH:mm", // ISO 8601 "MM/DD/YYYY hh:mm AM/PM", // US "DD.MM.YYYY", // date only "YYYY/MM/DD HH:mm:ss", // full timestamp } // defaultScanTitleDateFormat is the token pattern used when a tenant has none // set (empty column, e.g. legacy rows) — matches the historic default layout. const defaultScanTitleDateFormat = "DD.MM.YYYY HH:mm" // defaultScanTitlePrefix is the placeholder-title prefix used when a tenant has // none set (empty column, e.g. legacy rows) — matches the historic hard-coded // "Scan" prefix. const defaultScanTitlePrefix = "Scan" // scanTitleDateLayout translates a tenant's stored token pattern to its Go time // layout, falling back to the default for empty or invalid values so a bad // stored value can never break title generation. func scanTitleDateLayout(pattern string) string { if layout, err := dateformat.Translate(pattern); err == nil { return layout } layout, _ := dateformat.Translate(defaultScanTitleDateFormat) return layout } // titleFromOCRText derives a document title from the first non-empty, // non-trivial line of OCR text (e.g. a letterhead or invoice heading). // Falls back to a timestamp placeholder when the OCR text is empty or every // line is too short to be a meaningful title (stray punctuation, page // numbers, etc). dateLayout is a Go time layout (see scanTitleDateLayout) // selecting the placeholder timestamp format per tenant. // titleCandidateScanLines caps how many leading OCR lines are considered for // title derivation — real headings sit near the top of a document, and // limiting the scan avoids accidentally picking up a plausible-looking // fragment from deep in a noisy body. const titleCandidateScanLines = 8 // titleCandidateMinAlnumRatio filters out lines that are mostly OCR symbol // noise (e.g. "od?", stray punctuation) even though they clear the minimum // length — such lines tend to have a low ratio of letters/digits to total // non-space characters compared to a real heading. const titleCandidateMinAlnumRatio = 0.75 // isUsableTitleLine reports whether line looks like real text rather than // OCR noise: long enough, and made up predominantly of letters/digits. func isUsableTitleLine(line string) bool { if len(line) < 3 { return false } var total, alnum int for _, r := range line { if unicode.IsSpace(r) { continue } total++ if unicode.IsLetter(r) || unicode.IsDigit(r) { alnum++ } } if total == 0 { return false } return float64(alnum)/float64(total) >= titleCandidateMinAlnumRatio } func titleFromOCRText(ocrText, prefix, dateLayout string) string { lines := strings.Split(ocrText, "\n") if len(lines) > titleCandidateScanLines { lines = lines[:titleCandidateScanLines] } for _, line := range lines { line = strings.TrimSpace(line) if !isUsableTitleLine(line) { continue } r := []rune(line) if len(r) > maxDerivedTitleLen { line = string(r[:maxDerivedTitleLen]) } return line } if dateLayout == "" { dateLayout = scanTitleDateLayout(defaultScanTitleDateFormat) } if prefix == "" { prefix = defaultScanTitlePrefix } return fmt.Sprintf("%s %s", prefix, time.Now().Format(dateLayout)) } // tenantScanTitleParams loads the tenant's configured placeholder-title prefix // and date layout in a single lookup, falling back to the defaults on any // failure (best-effort — title generation must never fail an upload). func (s *Server) tenantScanTitleParams(ctx context.Context, tenantID int64) (prefix, dateLayout string) { if s.tenantStore == nil { return defaultScanTitlePrefix, scanTitleDateLayout(defaultScanTitleDateFormat) } t, err := s.tenantStore.GetByID(ctx, tenantID) if err != nil || t == nil { return defaultScanTitlePrefix, scanTitleDateLayout(defaultScanTitleDateFormat) } prefix = strings.TrimSpace(t.ScanTitlePrefix) if prefix == "" { prefix = defaultScanTitlePrefix } return prefix, scanTitleDateLayout(t.ScanTitleDateFormat) } func randomUploadID() string { b := make([]byte, 16) if _, err := rand.Read(b); err != nil { return fmt.Sprintf("upload-%d", time.Now().UnixNano()) } return hex.EncodeToString(b) } // copyFile is the cross-device fallback for os.Rename (EXDEV): copy + fsync // + remove the source. func copyFile(src, dst string) error { in, err := os.Open(src) if err != nil { return err } defer in.Close() out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o640) if err != nil { return err } if _, err := io.Copy(out, in); err != nil { out.Close() os.Remove(dst) return err } if err := out.Sync(); err != nil { out.Close() os.Remove(dst) return err } return out.Close() } // ocrSupportedMimeTypes is the whitelist of MIME types the OCR pipeline // (internal/ocr.Extract) actually dispatches on. A declared Content-Type is // only trusted verbatim when it names one of these; anything else (generic // application/octet-stream, empty, or a bogus value) triggers magic-byte / // extension fallback so a mislabeled but perfectly OCR-able upload still gets // text-extracted. var ocrSupportedMimeTypes = map[string]bool{ "application/pdf": true, "image/jpeg": true, "image/png": true, "image/tiff": true, "image/gif": true, "image/webp": true, "image/bmp": true, // E-Mail (parsed to text directly, see internal/ocr/convert.go). "message/rfc822": true, // Office documents (converted to PDF via LibreOffice before OCR). "application/msword": true, "application/vnd.openxmlformats-officedocument.wordprocessingml.document": true, "application/vnd.oasis.opendocument.text": true, "application/rtf": true, "text/rtf": true, "application/vnd.ms-excel": true, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true, "application/vnd.oasis.opendocument.spreadsheet": true, "application/vnd.ms-powerpoint": true, "application/vnd.openxmlformats-officedocument.presentationml.presentation": true, "application/vnd.oasis.opendocument.presentation": true, } // detectMimeType determines the MIME type used to drive OCR dispatch. It // prefers a declared Content-Type (may be empty, e.g. for SFTP uploads which // have no HTTP header) ONLY when it names a type the OCR pipeline understands. // Otherwise — the common "application/octet-stream" case from clients that // don't sniff, an empty header, or an unrecognized value — it sniffs the // file's magic bytes via http.DetectContentType (stdlib, first 512 bytes) and, // failing that, falls back to the file extension. filePath may be empty (magic // sniffing is then skipped). func detectMimeType(contentType, ext, filePath string) string { if ct := strings.SplitN(contentType, ";", 2)[0]; ocrSupportedMimeTypes[strings.ToLower(strings.TrimSpace(ct))] { return strings.ToLower(strings.TrimSpace(ct)) } // Magic-byte sniffing on the already-written inbox file. if filePath != "" { if sniffed := sniffMimeType(filePath); ocrSupportedMimeTypes[sniffed] { return sniffed } } // Extension whitelist fallback. switch strings.ToLower(ext) { case ".pdf": return "application/pdf" case ".jpg", ".jpeg": return "image/jpeg" case ".png": return "image/png" case ".tif", ".tiff": return "image/tiff" case ".gif": return "image/gif" case ".webp": return "image/webp" case ".bmp": return "image/bmp" case ".eml": return "message/rfc822" case ".doc": return "application/msword" case ".docx": return "application/vnd.openxmlformats-officedocument.wordprocessingml.document" case ".odt": return "application/vnd.oasis.opendocument.text" case ".rtf": return "application/rtf" case ".xls": return "application/vnd.ms-excel" case ".xlsx": return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" case ".ods": return "application/vnd.oasis.opendocument.spreadsheet" case ".ppt": return "application/vnd.ms-powerpoint" case ".pptx": return "application/vnd.openxmlformats-officedocument.presentationml.presentation" case ".odp": return "application/vnd.oasis.opendocument.presentation" default: return "application/octet-stream" } } // sniffMimeType reads the first 512 bytes of filePath and classifies them via // http.DetectContentType. Returns "" on any read error. The declared type is // stripped of parameters (http.DetectContentType may append "; charset=..."). func sniffMimeType(filePath string) string { f, err := os.Open(filePath) if err != nil { return "" } defer f.Close() buf := make([]byte, 512) n, err := io.ReadFull(f, buf) if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF { return "" } ct := http.DetectContentType(buf[:n]) return strings.ToLower(strings.TrimSpace(strings.SplitN(ct, ";", 2)[0])) }