patrick
89de794356
- FDN-02: Rollback-fähige Down-Migrationen (024-026), archivdms seed dev CLI - FDN-03: internal/objectstore Interface + lokaler WORM-Treiber, signierte Download-URLs - FDN-07: go.mod/go.sum vervollständigt (fehlender go-ldap/v3-Eintrag), CI-Pipeline (.gitea/workflows/ci.yml, bereits in FDN-01 committet) damit lauffähig - FDN-08: Request-ID-Middleware, /metrics-Endpoint, Panic-Recovery, Login/Logout/Me technisches Logging inkl. Access-Log je Anfrage
427 lines
14 KiB
Go
427 lines
14 KiB
Go
package api
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"encoding/csv"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"archivdms/internal/audit"
|
|
"archivdms/internal/auth"
|
|
"archivdms/internal/storage"
|
|
"archivdms/internal/userstore"
|
|
)
|
|
|
|
// maxBulkExportDocuments caps how many documents a single bulk export may
|
|
// stream. Explicit ID lists beyond this are rejected with 400; a filter
|
|
// selection resolving to more documents is likewise rejected so the caller
|
|
// narrows the filter instead of silently receiving a truncated archive
|
|
// (GoBD-Vollständigkeit: a partial export must never look complete).
|
|
const maxBulkExportDocuments = 500
|
|
|
|
// bulkExportRequest is the POST /api/documents/export body. Either IDs or
|
|
// Filter is used — IDs takes precedence when both are present.
|
|
//
|
|
// {"ids": [1,2,3]}
|
|
// {"filter": {"doc_type_id": 4, "tag_ids": [7,9],
|
|
// "document_date_from": "2026-01-01", "document_date_to": "2026-03-31"}}
|
|
type bulkExportRequest struct {
|
|
IDs []int64 `json:"ids"`
|
|
Filter *bulkExportFilter `json:"filter"`
|
|
}
|
|
|
|
// bulkExportFilter mirrors the filter dimensions the list/search endpoints
|
|
// already expose (doc type, correspondent, tags, time range). It is applied on
|
|
// top of the tenant- and ACL-scoped result of Store.ListDocuments, so no new
|
|
// SQL predicate — and no new place a tenant_id filter could be forgotten.
|
|
type bulkExportFilter struct {
|
|
DocTypeID *int64 `json:"doc_type_id"`
|
|
CorrespondentID *int64 `json:"correspondent_id"`
|
|
TagIDs []int64 `json:"tag_ids"`
|
|
DocumentDateFrom string `json:"document_date_from"` // YYYY-MM-DD, inclusive
|
|
DocumentDateTo string `json:"document_date_to"` // YYYY-MM-DD, inclusive
|
|
UploadedFrom string `json:"uploaded_from"` // YYYY-MM-DD, inclusive
|
|
UploadedTo string `json:"uploaded_to"` // YYYY-MM-DD, inclusive (whole day)
|
|
}
|
|
|
|
// bulkExportCSVHeader is the index.csv header. Column names are deliberately
|
|
// identical to the metadata.json field names (snake_case) so the later DATEV
|
|
// formatter can map from one shared vocabulary.
|
|
var bulkExportCSVHeader = []string{
|
|
"document_id", "title", "doc_type", "correspondent",
|
|
"document_date", "tags", "uploaded_at",
|
|
}
|
|
|
|
// handleBulkExportDocuments streams a multi-document ZIP
|
|
// (POST /api/documents/export):
|
|
//
|
|
// doc-<id>/<title>.<ext> original WORM file
|
|
// doc-<id>/metadata.json same shape as the single-document export
|
|
// doc-<id>/ocr_text.txt only when OCR text exists
|
|
// index.csv one row per exported document
|
|
// errors.txt only when documents were skipped
|
|
//
|
|
// ACL: identical to the single export — tenant scoping plus, for role 'user',
|
|
// the per-document document_visibility check. Documents the caller may not see
|
|
// (or that fail to read) are skipped and listed in errors.txt rather than
|
|
// aborting the whole request.
|
|
//
|
|
// Audit: exactly ONE EventDocumentBulkExport entry per call, carrying the
|
|
// exported/skipped counts.
|
|
func (s *Server) handleBulkExportDocuments(w http.ResponseWriter, r *http.Request) {
|
|
sess := sessionFromCtx(r.Context())
|
|
if sess.TenantID == nil {
|
|
writeError(w, http.StatusForbidden, "tenant context required")
|
|
return
|
|
}
|
|
tenantID := *sess.TenantID
|
|
|
|
fail := func(status int, msg, detail string) {
|
|
s.audlog.Log(audit.Entry{
|
|
EventType: audit.EventDocumentBulkExport, Username: sess.Username, TenantID: sess.TenantID,
|
|
Success: false, Detail: detail,
|
|
})
|
|
writeError(w, status, msg)
|
|
}
|
|
|
|
var req bulkExportRequest
|
|
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
|
|
fail(http.StatusBadRequest, "invalid request body", "decode_failed: "+err.Error())
|
|
return
|
|
}
|
|
if len(req.IDs) == 0 && req.Filter == nil {
|
|
fail(http.StatusBadRequest, "ids oder filter erforderlich", "empty_selection")
|
|
return
|
|
}
|
|
if len(req.IDs) > maxBulkExportDocuments {
|
|
fail(http.StatusBadRequest,
|
|
fmt.Sprintf("maximal %d Dokumente pro Export (angefragt: %d)", maxBulkExportDocuments, len(req.IDs)),
|
|
fmt.Sprintf("too_many_ids: %d", len(req.IDs)))
|
|
return
|
|
}
|
|
|
|
// Role 'user' gets the group-resolved ACL applied by the store; domain
|
|
// admins and superadmins see the whole tenant (roles are the outer boundary).
|
|
var aclUserID *int64
|
|
if !auth.HasRole(sess.Role, userstore.RoleDomainAdmin) {
|
|
uid := sess.UserID
|
|
aclUserID = &uid
|
|
}
|
|
|
|
// skipped collects IDs that were requested but not exported, with a reason.
|
|
skipped := make([]string, 0, 8)
|
|
|
|
// Resolve the selection into a concrete, ordered document list.
|
|
docs := make([]storage.Document, 0, len(req.IDs))
|
|
if len(req.IDs) > 0 {
|
|
for _, id := range req.IDs {
|
|
doc, err := s.store.GetDocument(r.Context(), id, tenantID)
|
|
if err != nil || doc == nil {
|
|
skipped = append(skipped, fmt.Sprintf("%d: nicht gefunden", id))
|
|
continue
|
|
}
|
|
if aclUserID != nil {
|
|
visible, err := s.store.IsDocumentVisible(r.Context(), id, tenantID, *aclUserID)
|
|
if err != nil {
|
|
skipped = append(skipped, fmt.Sprintf("%d: Sichtbarkeitsprüfung fehlgeschlagen", id))
|
|
continue
|
|
}
|
|
if !visible {
|
|
skipped = append(skipped, fmt.Sprintf("%d: nicht sichtbar", id))
|
|
continue
|
|
}
|
|
}
|
|
docs = append(docs, *doc)
|
|
}
|
|
} else {
|
|
all, err := s.store.ListDocuments(r.Context(), tenantID, aclUserID)
|
|
if err != nil {
|
|
fail(http.StatusInternalServerError, "export failed", "list_failed: "+err.Error())
|
|
return
|
|
}
|
|
filtered, err := s.filterBulkExportDocs(r, tenantID, all, req.Filter)
|
|
if err != nil {
|
|
fail(http.StatusBadRequest, err.Error(), "filter_invalid: "+err.Error())
|
|
return
|
|
}
|
|
if len(filtered) > maxBulkExportDocuments {
|
|
fail(http.StatusBadRequest,
|
|
fmt.Sprintf("Filter trifft %d Dokumente, maximal %d pro Export — Filter eingrenzen", len(filtered), maxBulkExportDocuments),
|
|
fmt.Sprintf("filter_too_broad: %d", len(filtered)))
|
|
return
|
|
}
|
|
docs = filtered
|
|
}
|
|
|
|
if len(docs) == 0 && len(skipped) == 0 {
|
|
fail(http.StatusNotFound, "keine Dokumente für den Export gefunden", "empty_result")
|
|
return
|
|
}
|
|
|
|
ts := time.Now().UTC().Format("20060102-150405")
|
|
w.Header().Set("Content-Type", "application/zip")
|
|
w.Header().Set("Content-Disposition", "attachment; filename=\"export-bulk-"+ts+".zip\"")
|
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
w.WriteHeader(http.StatusOK)
|
|
|
|
zw := zip.NewWriter(w)
|
|
writeEntry := func(name string, rd io.Reader) error {
|
|
entry, err := zw.Create(name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = io.Copy(entry, rd)
|
|
return err
|
|
}
|
|
|
|
csvBuf := &bytes.Buffer{}
|
|
// UTF-8 BOM so Excel opens the CSV with correct umlauts.
|
|
csvBuf.WriteString("\xef\xbb\xbf")
|
|
cw := csv.NewWriter(csvBuf)
|
|
cw.Comma = ';'
|
|
_ = cw.Write(bulkExportCSVHeader)
|
|
|
|
exported := 0
|
|
var streamErr error
|
|
for i := range docs {
|
|
doc := docs[i]
|
|
row, err := s.writeBulkExportDoc(r, writeEntry, sess.Username, tenantID, &doc)
|
|
if err != nil {
|
|
// A single unreadable document must not kill the archive — unless the
|
|
// ZIP writer itself failed, which we detect on Close below.
|
|
s.reqLog(r.Context()).Warn("bulk export: document skipped", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
|
skipped = append(skipped, fmt.Sprintf("%d: %v", doc.ID, err))
|
|
continue
|
|
}
|
|
_ = cw.Write(row)
|
|
exported++
|
|
}
|
|
cw.Flush()
|
|
|
|
if streamErr == nil {
|
|
streamErr = writeEntry("index.csv", bytes.NewReader(csvBuf.Bytes()))
|
|
}
|
|
if streamErr == nil && len(skipped) > 0 {
|
|
var b strings.Builder
|
|
b.WriteString("Übersprungene Dokumente (nicht sichtbar, nicht gefunden oder Lesefehler):\n")
|
|
for _, line := range skipped {
|
|
b.WriteString(line)
|
|
b.WriteString("\n")
|
|
}
|
|
streamErr = writeEntry("errors.txt", strings.NewReader(b.String()))
|
|
}
|
|
if closeErr := zw.Close(); streamErr == nil {
|
|
streamErr = closeErr
|
|
}
|
|
|
|
detail := fmt.Sprintf("zip_bulk_export: exported=%d skipped=%d", exported, len(skipped))
|
|
if streamErr != nil {
|
|
// Headers are already out — audit the partial export, no HTTP error.
|
|
s.reqLog(r.Context()).Warn("bulk document export stream failed", "tenant_id", tenantID, "err", streamErr)
|
|
s.audlog.Log(audit.Entry{
|
|
EventType: audit.EventDocumentBulkExport, Username: sess.Username, TenantID: sess.TenantID,
|
|
Success: false, Detail: detail + " stream_failed: " + streamErr.Error(),
|
|
})
|
|
return
|
|
}
|
|
s.audlog.Log(audit.Entry{
|
|
EventType: audit.EventDocumentBulkExport, Username: sess.Username, TenantID: sess.TenantID,
|
|
Success: true, Detail: detail,
|
|
})
|
|
}
|
|
|
|
// writeBulkExportDoc writes the doc-<id>/ folder of one document and returns
|
|
// its index.csv row. Any error means "skip this document" — the caller keeps
|
|
// the archive going and records the ID in errors.txt.
|
|
func (s *Server) writeBulkExportDoc(
|
|
r *http.Request,
|
|
writeEntry func(string, io.Reader) error,
|
|
username string,
|
|
tenantID int64,
|
|
doc *storage.Document,
|
|
) ([]string, error) {
|
|
ctx := r.Context()
|
|
prefix := "doc-" + strconv.FormatInt(doc.ID, 10) + "/"
|
|
|
|
docTypeName, correspondentName, err := s.store.DocumentTaxonomyNames(ctx, doc.ID, tenantID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Taxonomie nicht lesbar: %w", err)
|
|
}
|
|
// Bestandsschutz: fall back to the deprecated free-text columns.
|
|
if docTypeName == "" {
|
|
docTypeName = doc.DocType
|
|
}
|
|
if correspondentName == "" {
|
|
correspondentName = doc.Correspondent
|
|
}
|
|
|
|
tagEntities, err := s.store.ListDocumentTags(ctx, doc.ID, tenantID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Tags nicht lesbar: %w", err)
|
|
}
|
|
tags := make([]string, 0, len(tagEntities))
|
|
for _, t := range tagEntities {
|
|
tags = append(tags, t.Name)
|
|
}
|
|
|
|
fieldValues, err := s.store.ListDocumentFieldValues(ctx, doc.ID, tenantID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Zusatzfelder nicht lesbar: %w", err)
|
|
}
|
|
|
|
f, err := os.Open(doc.StoragePath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Datei nicht lesbar: %w", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
meta := documentExportMetadata{
|
|
DocumentID: doc.ID,
|
|
TenantID: doc.TenantID,
|
|
Title: doc.Title,
|
|
DocType: docTypeName,
|
|
Correspondent: correspondentName,
|
|
Tags: tags,
|
|
DocumentDateScore: doc.DocumentDateScore,
|
|
UploadedAt: doc.CreatedAt,
|
|
UpdatedAt: doc.UpdatedAt,
|
|
CreatedBy: s.exportCreatorName(doc),
|
|
ContentHash: doc.ContentHash,
|
|
OriginalFilename: filepath.Base(doc.StoragePath),
|
|
CustomFields: exportCustomFields(fieldValues),
|
|
ExportedAt: time.Now().UTC(),
|
|
ExportedBy: username,
|
|
}
|
|
docDate := ""
|
|
if doc.DocumentDate != nil {
|
|
docDate = doc.DocumentDate.Format("2006-01-02")
|
|
meta.DocumentDate = &docDate
|
|
}
|
|
if doc.RetainUntil != nil {
|
|
d := doc.RetainUntil.Format("2006-01-02")
|
|
meta.RetainUntil = &d
|
|
}
|
|
metaJSON, err := json.MarshalIndent(meta, "", " ")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Metadaten nicht serialisierbar: %w", err)
|
|
}
|
|
|
|
ext := filepath.Ext(doc.StoragePath)
|
|
if err := writeEntry(prefix+safeDownloadName(doc.Title, ext), f); err != nil {
|
|
return nil, fmt.Errorf("ZIP-Eintrag fehlgeschlagen: %w", err)
|
|
}
|
|
if err := writeEntry(prefix+"metadata.json", bytes.NewReader(metaJSON)); err != nil {
|
|
return nil, fmt.Errorf("ZIP-Eintrag fehlgeschlagen: %w", err)
|
|
}
|
|
if doc.OCRText != "" {
|
|
if err := writeEntry(prefix+"ocr_text.txt", strings.NewReader(doc.OCRText)); err != nil {
|
|
return nil, fmt.Errorf("ZIP-Eintrag fehlgeschlagen: %w", err)
|
|
}
|
|
}
|
|
|
|
return []string{
|
|
strconv.FormatInt(doc.ID, 10),
|
|
doc.Title,
|
|
docTypeName,
|
|
correspondentName,
|
|
docDate,
|
|
strings.Join(tags, ", "),
|
|
doc.CreatedAt.UTC().Format(time.RFC3339),
|
|
}, nil
|
|
}
|
|
|
|
// filterBulkExportDocs narrows an already tenant- and ACL-scoped document list
|
|
// by the requested filter. Tag filtering needs a per-document lookup, so it is
|
|
// applied last, after the cheap in-memory predicates.
|
|
func (s *Server) filterBulkExportDocs(r *http.Request, tenantID int64, docs []storage.Document, f *bulkExportFilter) ([]storage.Document, error) {
|
|
docFrom, err := parseBulkExportDate(f.DocumentDateFrom)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ungültiges document_date_from (erwartet YYYY-MM-DD)")
|
|
}
|
|
docTo, err := parseBulkExportDate(f.DocumentDateTo)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ungültiges document_date_to (erwartet YYYY-MM-DD)")
|
|
}
|
|
upFrom, err := parseBulkExportDate(f.UploadedFrom)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ungültiges uploaded_from (erwartet YYYY-MM-DD)")
|
|
}
|
|
upTo, err := parseBulkExportDate(f.UploadedTo)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ungültiges uploaded_to (erwartet YYYY-MM-DD)")
|
|
}
|
|
|
|
out := make([]storage.Document, 0, len(docs))
|
|
for i := range docs {
|
|
d := docs[i]
|
|
if f.DocTypeID != nil && (d.DocTypeID == nil || *d.DocTypeID != *f.DocTypeID) {
|
|
continue
|
|
}
|
|
if f.CorrespondentID != nil && (d.CorrespondentID == nil || *d.CorrespondentID != *f.CorrespondentID) {
|
|
continue
|
|
}
|
|
if docFrom != nil || docTo != nil {
|
|
if d.DocumentDate == nil {
|
|
continue
|
|
}
|
|
day := d.DocumentDate.UTC().Truncate(24 * time.Hour)
|
|
if docFrom != nil && day.Before(*docFrom) {
|
|
continue
|
|
}
|
|
if docTo != nil && day.After(*docTo) {
|
|
continue
|
|
}
|
|
}
|
|
if upFrom != nil && d.CreatedAt.UTC().Before(*upFrom) {
|
|
continue
|
|
}
|
|
if upTo != nil && d.CreatedAt.UTC().After(upTo.Add(24*time.Hour-time.Nanosecond)) {
|
|
continue
|
|
}
|
|
out = append(out, d)
|
|
}
|
|
|
|
if len(f.TagIDs) == 0 {
|
|
return out, nil
|
|
}
|
|
want := make(map[int64]struct{}, len(f.TagIDs))
|
|
for _, id := range f.TagIDs {
|
|
want[id] = struct{}{}
|
|
}
|
|
tagged := make([]storage.Document, 0, len(out))
|
|
for i := range out {
|
|
tags, err := s.store.ListDocumentTags(r.Context(), out[i].ID, tenantID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Tag-Filter fehlgeschlagen")
|
|
}
|
|
for _, t := range tags {
|
|
if _, ok := want[t.ID]; ok {
|
|
tagged = append(tagged, out[i])
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return tagged, nil
|
|
}
|
|
|
|
// parseBulkExportDate parses an optional YYYY-MM-DD filter bound (UTC).
|
|
func parseBulkExportDate(s string) (*time.Time, error) {
|
|
s = strings.TrimSpace(s)
|
|
if s == "" {
|
|
return nil, nil
|
|
}
|
|
t, err := time.ParseInLocation("2006-01-02", s, time.UTC)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parse date %q: %w", s, err)
|
|
}
|
|
return &t, nil
|
|
}
|