FDN-01: repository & projektgerüst
Git-Repository für bestehenden archivdms-Code initialisiert, Branch-/Commit-Konvention (feature/<ticket>-<slug>-Branches, Ticket-Prefix in Commit-Nachricht) etabliert.
This commit is contained in:
@@ -0,0 +1,426 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archivdms/internal/audit"
|
||||
"archivdms/internal/auth"
|
||||
"archivdms/internal/storage"
|
||||
"archivdms/internal/userstore"
|
||||
)
|
||||
|
||||
// maxBulkExportDocuments caps how many documents a single bulk export may
|
||||
// stream. Explicit ID lists beyond this are rejected with 400; a filter
|
||||
// selection resolving to more documents is likewise rejected so the caller
|
||||
// narrows the filter instead of silently receiving a truncated archive
|
||||
// (GoBD-Vollständigkeit: a partial export must never look complete).
|
||||
const maxBulkExportDocuments = 500
|
||||
|
||||
// bulkExportRequest is the POST /api/documents/export body. Either IDs or
|
||||
// Filter is used — IDs takes precedence when both are present.
|
||||
//
|
||||
// {"ids": [1,2,3]}
|
||||
// {"filter": {"doc_type_id": 4, "tag_ids": [7,9],
|
||||
// "document_date_from": "2026-01-01", "document_date_to": "2026-03-31"}}
|
||||
type bulkExportRequest struct {
|
||||
IDs []int64 `json:"ids"`
|
||||
Filter *bulkExportFilter `json:"filter"`
|
||||
}
|
||||
|
||||
// bulkExportFilter mirrors the filter dimensions the list/search endpoints
|
||||
// already expose (doc type, correspondent, tags, time range). It is applied on
|
||||
// top of the tenant- and ACL-scoped result of Store.ListDocuments, so no new
|
||||
// SQL predicate — and no new place a tenant_id filter could be forgotten.
|
||||
type bulkExportFilter struct {
|
||||
DocTypeID *int64 `json:"doc_type_id"`
|
||||
CorrespondentID *int64 `json:"correspondent_id"`
|
||||
TagIDs []int64 `json:"tag_ids"`
|
||||
DocumentDateFrom string `json:"document_date_from"` // YYYY-MM-DD, inclusive
|
||||
DocumentDateTo string `json:"document_date_to"` // YYYY-MM-DD, inclusive
|
||||
UploadedFrom string `json:"uploaded_from"` // YYYY-MM-DD, inclusive
|
||||
UploadedTo string `json:"uploaded_to"` // YYYY-MM-DD, inclusive (whole day)
|
||||
}
|
||||
|
||||
// bulkExportCSVHeader is the index.csv header. Column names are deliberately
|
||||
// identical to the metadata.json field names (snake_case) so the later DATEV
|
||||
// formatter can map from one shared vocabulary.
|
||||
var bulkExportCSVHeader = []string{
|
||||
"document_id", "title", "doc_type", "correspondent",
|
||||
"document_date", "tags", "uploaded_at",
|
||||
}
|
||||
|
||||
// handleBulkExportDocuments streams a multi-document ZIP
|
||||
// (POST /api/documents/export):
|
||||
//
|
||||
// doc-<id>/<title>.<ext> original WORM file
|
||||
// doc-<id>/metadata.json same shape as the single-document export
|
||||
// doc-<id>/ocr_text.txt only when OCR text exists
|
||||
// index.csv one row per exported document
|
||||
// errors.txt only when documents were skipped
|
||||
//
|
||||
// ACL: identical to the single export — tenant scoping plus, for role 'user',
|
||||
// the per-document document_visibility check. Documents the caller may not see
|
||||
// (or that fail to read) are skipped and listed in errors.txt rather than
|
||||
// aborting the whole request.
|
||||
//
|
||||
// Audit: exactly ONE EventDocumentBulkExport entry per call, carrying the
|
||||
// exported/skipped counts.
|
||||
func (s *Server) handleBulkExportDocuments(w http.ResponseWriter, r *http.Request) {
|
||||
sess := sessionFromCtx(r.Context())
|
||||
if sess.TenantID == nil {
|
||||
writeError(w, http.StatusForbidden, "tenant context required")
|
||||
return
|
||||
}
|
||||
tenantID := *sess.TenantID
|
||||
|
||||
fail := func(status int, msg, detail string) {
|
||||
s.audlog.Log(audit.Entry{
|
||||
EventType: audit.EventDocumentBulkExport, Username: sess.Username, TenantID: sess.TenantID,
|
||||
Success: false, Detail: detail,
|
||||
})
|
||||
writeError(w, status, msg)
|
||||
}
|
||||
|
||||
var req bulkExportRequest
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
|
||||
fail(http.StatusBadRequest, "invalid request body", "decode_failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
if len(req.IDs) == 0 && req.Filter == nil {
|
||||
fail(http.StatusBadRequest, "ids oder filter erforderlich", "empty_selection")
|
||||
return
|
||||
}
|
||||
if len(req.IDs) > maxBulkExportDocuments {
|
||||
fail(http.StatusBadRequest,
|
||||
fmt.Sprintf("maximal %d Dokumente pro Export (angefragt: %d)", maxBulkExportDocuments, len(req.IDs)),
|
||||
fmt.Sprintf("too_many_ids: %d", len(req.IDs)))
|
||||
return
|
||||
}
|
||||
|
||||
// Role 'user' gets the group-resolved ACL applied by the store; domain
|
||||
// admins and superadmins see the whole tenant (roles are the outer boundary).
|
||||
var aclUserID *int64
|
||||
if !auth.HasRole(sess.Role, userstore.RoleDomainAdmin) {
|
||||
uid := sess.UserID
|
||||
aclUserID = &uid
|
||||
}
|
||||
|
||||
// skipped collects IDs that were requested but not exported, with a reason.
|
||||
skipped := make([]string, 0, 8)
|
||||
|
||||
// Resolve the selection into a concrete, ordered document list.
|
||||
docs := make([]storage.Document, 0, len(req.IDs))
|
||||
if len(req.IDs) > 0 {
|
||||
for _, id := range req.IDs {
|
||||
doc, err := s.store.GetDocument(r.Context(), id, tenantID)
|
||||
if err != nil || doc == nil {
|
||||
skipped = append(skipped, fmt.Sprintf("%d: nicht gefunden", id))
|
||||
continue
|
||||
}
|
||||
if aclUserID != nil {
|
||||
visible, err := s.store.IsDocumentVisible(r.Context(), id, tenantID, *aclUserID)
|
||||
if err != nil {
|
||||
skipped = append(skipped, fmt.Sprintf("%d: Sichtbarkeitsprüfung fehlgeschlagen", id))
|
||||
continue
|
||||
}
|
||||
if !visible {
|
||||
skipped = append(skipped, fmt.Sprintf("%d: nicht sichtbar", id))
|
||||
continue
|
||||
}
|
||||
}
|
||||
docs = append(docs, *doc)
|
||||
}
|
||||
} else {
|
||||
all, err := s.store.ListDocuments(r.Context(), tenantID, aclUserID)
|
||||
if err != nil {
|
||||
fail(http.StatusInternalServerError, "export failed", "list_failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
filtered, err := s.filterBulkExportDocs(r, tenantID, all, req.Filter)
|
||||
if err != nil {
|
||||
fail(http.StatusBadRequest, err.Error(), "filter_invalid: "+err.Error())
|
||||
return
|
||||
}
|
||||
if len(filtered) > maxBulkExportDocuments {
|
||||
fail(http.StatusBadRequest,
|
||||
fmt.Sprintf("Filter trifft %d Dokumente, maximal %d pro Export — Filter eingrenzen", len(filtered), maxBulkExportDocuments),
|
||||
fmt.Sprintf("filter_too_broad: %d", len(filtered)))
|
||||
return
|
||||
}
|
||||
docs = filtered
|
||||
}
|
||||
|
||||
if len(docs) == 0 && len(skipped) == 0 {
|
||||
fail(http.StatusNotFound, "keine Dokumente für den Export gefunden", "empty_result")
|
||||
return
|
||||
}
|
||||
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
w.Header().Set("Content-Disposition", "attachment; filename=\"export-bulk-"+ts+".zip\"")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
zw := zip.NewWriter(w)
|
||||
writeEntry := func(name string, rd io.Reader) error {
|
||||
entry, err := zw.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(entry, rd)
|
||||
return err
|
||||
}
|
||||
|
||||
csvBuf := &bytes.Buffer{}
|
||||
// UTF-8 BOM so Excel opens the CSV with correct umlauts.
|
||||
csvBuf.WriteString("\xef\xbb\xbf")
|
||||
cw := csv.NewWriter(csvBuf)
|
||||
cw.Comma = ';'
|
||||
_ = cw.Write(bulkExportCSVHeader)
|
||||
|
||||
exported := 0
|
||||
var streamErr error
|
||||
for i := range docs {
|
||||
doc := docs[i]
|
||||
row, err := s.writeBulkExportDoc(r, writeEntry, sess.Username, tenantID, &doc)
|
||||
if err != nil {
|
||||
// A single unreadable document must not kill the archive — unless the
|
||||
// ZIP writer itself failed, which we detect on Close below.
|
||||
s.logger.Warn("bulk export: document skipped", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
|
||||
skipped = append(skipped, fmt.Sprintf("%d: %v", doc.ID, err))
|
||||
continue
|
||||
}
|
||||
_ = cw.Write(row)
|
||||
exported++
|
||||
}
|
||||
cw.Flush()
|
||||
|
||||
if streamErr == nil {
|
||||
streamErr = writeEntry("index.csv", bytes.NewReader(csvBuf.Bytes()))
|
||||
}
|
||||
if streamErr == nil && len(skipped) > 0 {
|
||||
var b strings.Builder
|
||||
b.WriteString("Übersprungene Dokumente (nicht sichtbar, nicht gefunden oder Lesefehler):\n")
|
||||
for _, line := range skipped {
|
||||
b.WriteString(line)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
streamErr = writeEntry("errors.txt", strings.NewReader(b.String()))
|
||||
}
|
||||
if closeErr := zw.Close(); streamErr == nil {
|
||||
streamErr = closeErr
|
||||
}
|
||||
|
||||
detail := fmt.Sprintf("zip_bulk_export: exported=%d skipped=%d", exported, len(skipped))
|
||||
if streamErr != nil {
|
||||
// Headers are already out — audit the partial export, no HTTP error.
|
||||
s.logger.Warn("bulk document export stream failed", "tenant_id", tenantID, "err", streamErr)
|
||||
s.audlog.Log(audit.Entry{
|
||||
EventType: audit.EventDocumentBulkExport, Username: sess.Username, TenantID: sess.TenantID,
|
||||
Success: false, Detail: detail + " stream_failed: " + streamErr.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
s.audlog.Log(audit.Entry{
|
||||
EventType: audit.EventDocumentBulkExport, Username: sess.Username, TenantID: sess.TenantID,
|
||||
Success: true, Detail: detail,
|
||||
})
|
||||
}
|
||||
|
||||
// writeBulkExportDoc writes the doc-<id>/ folder of one document and returns
|
||||
// its index.csv row. Any error means "skip this document" — the caller keeps
|
||||
// the archive going and records the ID in errors.txt.
|
||||
func (s *Server) writeBulkExportDoc(
|
||||
r *http.Request,
|
||||
writeEntry func(string, io.Reader) error,
|
||||
username string,
|
||||
tenantID int64,
|
||||
doc *storage.Document,
|
||||
) ([]string, error) {
|
||||
ctx := r.Context()
|
||||
prefix := "doc-" + strconv.FormatInt(doc.ID, 10) + "/"
|
||||
|
||||
docTypeName, correspondentName, err := s.store.DocumentTaxonomyNames(ctx, doc.ID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Taxonomie nicht lesbar: %w", err)
|
||||
}
|
||||
// Bestandsschutz: fall back to the deprecated free-text columns.
|
||||
if docTypeName == "" {
|
||||
docTypeName = doc.DocType
|
||||
}
|
||||
if correspondentName == "" {
|
||||
correspondentName = doc.Correspondent
|
||||
}
|
||||
|
||||
tagEntities, err := s.store.ListDocumentTags(ctx, doc.ID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Tags nicht lesbar: %w", err)
|
||||
}
|
||||
tags := make([]string, 0, len(tagEntities))
|
||||
for _, t := range tagEntities {
|
||||
tags = append(tags, t.Name)
|
||||
}
|
||||
|
||||
fieldValues, err := s.store.ListDocumentFieldValues(ctx, doc.ID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Zusatzfelder nicht lesbar: %w", err)
|
||||
}
|
||||
|
||||
f, err := os.Open(doc.StoragePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Datei nicht lesbar: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
meta := documentExportMetadata{
|
||||
DocumentID: doc.ID,
|
||||
TenantID: doc.TenantID,
|
||||
Title: doc.Title,
|
||||
DocType: docTypeName,
|
||||
Correspondent: correspondentName,
|
||||
Tags: tags,
|
||||
DocumentDateScore: doc.DocumentDateScore,
|
||||
UploadedAt: doc.CreatedAt,
|
||||
UpdatedAt: doc.UpdatedAt,
|
||||
CreatedBy: s.exportCreatorName(doc),
|
||||
ContentHash: doc.ContentHash,
|
||||
OriginalFilename: filepath.Base(doc.StoragePath),
|
||||
CustomFields: exportCustomFields(fieldValues),
|
||||
ExportedAt: time.Now().UTC(),
|
||||
ExportedBy: username,
|
||||
}
|
||||
docDate := ""
|
||||
if doc.DocumentDate != nil {
|
||||
docDate = doc.DocumentDate.Format("2006-01-02")
|
||||
meta.DocumentDate = &docDate
|
||||
}
|
||||
if doc.RetainUntil != nil {
|
||||
d := doc.RetainUntil.Format("2006-01-02")
|
||||
meta.RetainUntil = &d
|
||||
}
|
||||
metaJSON, err := json.MarshalIndent(meta, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Metadaten nicht serialisierbar: %w", err)
|
||||
}
|
||||
|
||||
ext := filepath.Ext(doc.StoragePath)
|
||||
if err := writeEntry(prefix+safeDownloadName(doc.Title, ext), f); err != nil {
|
||||
return nil, fmt.Errorf("ZIP-Eintrag fehlgeschlagen: %w", err)
|
||||
}
|
||||
if err := writeEntry(prefix+"metadata.json", bytes.NewReader(metaJSON)); err != nil {
|
||||
return nil, fmt.Errorf("ZIP-Eintrag fehlgeschlagen: %w", err)
|
||||
}
|
||||
if doc.OCRText != "" {
|
||||
if err := writeEntry(prefix+"ocr_text.txt", strings.NewReader(doc.OCRText)); err != nil {
|
||||
return nil, fmt.Errorf("ZIP-Eintrag fehlgeschlagen: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return []string{
|
||||
strconv.FormatInt(doc.ID, 10),
|
||||
doc.Title,
|
||||
docTypeName,
|
||||
correspondentName,
|
||||
docDate,
|
||||
strings.Join(tags, ", "),
|
||||
doc.CreatedAt.UTC().Format(time.RFC3339),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// filterBulkExportDocs narrows an already tenant- and ACL-scoped document list
|
||||
// by the requested filter. Tag filtering needs a per-document lookup, so it is
|
||||
// applied last, after the cheap in-memory predicates.
|
||||
func (s *Server) filterBulkExportDocs(r *http.Request, tenantID int64, docs []storage.Document, f *bulkExportFilter) ([]storage.Document, error) {
|
||||
docFrom, err := parseBulkExportDate(f.DocumentDateFrom)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ungültiges document_date_from (erwartet YYYY-MM-DD)")
|
||||
}
|
||||
docTo, err := parseBulkExportDate(f.DocumentDateTo)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ungültiges document_date_to (erwartet YYYY-MM-DD)")
|
||||
}
|
||||
upFrom, err := parseBulkExportDate(f.UploadedFrom)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ungültiges uploaded_from (erwartet YYYY-MM-DD)")
|
||||
}
|
||||
upTo, err := parseBulkExportDate(f.UploadedTo)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ungültiges uploaded_to (erwartet YYYY-MM-DD)")
|
||||
}
|
||||
|
||||
out := make([]storage.Document, 0, len(docs))
|
||||
for i := range docs {
|
||||
d := docs[i]
|
||||
if f.DocTypeID != nil && (d.DocTypeID == nil || *d.DocTypeID != *f.DocTypeID) {
|
||||
continue
|
||||
}
|
||||
if f.CorrespondentID != nil && (d.CorrespondentID == nil || *d.CorrespondentID != *f.CorrespondentID) {
|
||||
continue
|
||||
}
|
||||
if docFrom != nil || docTo != nil {
|
||||
if d.DocumentDate == nil {
|
||||
continue
|
||||
}
|
||||
day := d.DocumentDate.UTC().Truncate(24 * time.Hour)
|
||||
if docFrom != nil && day.Before(*docFrom) {
|
||||
continue
|
||||
}
|
||||
if docTo != nil && day.After(*docTo) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if upFrom != nil && d.CreatedAt.UTC().Before(*upFrom) {
|
||||
continue
|
||||
}
|
||||
if upTo != nil && d.CreatedAt.UTC().After(upTo.Add(24*time.Hour-time.Nanosecond)) {
|
||||
continue
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
|
||||
if len(f.TagIDs) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
want := make(map[int64]struct{}, len(f.TagIDs))
|
||||
for _, id := range f.TagIDs {
|
||||
want[id] = struct{}{}
|
||||
}
|
||||
tagged := make([]storage.Document, 0, len(out))
|
||||
for i := range out {
|
||||
tags, err := s.store.ListDocumentTags(r.Context(), out[i].ID, tenantID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Tag-Filter fehlgeschlagen")
|
||||
}
|
||||
for _, t := range tags {
|
||||
if _, ok := want[t.ID]; ok {
|
||||
tagged = append(tagged, out[i])
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return tagged, nil
|
||||
}
|
||||
|
||||
// parseBulkExportDate parses an optional YYYY-MM-DD filter bound (UTC).
|
||||
func parseBulkExportDate(s string) (*time.Time, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return nil, nil
|
||||
}
|
||||
t, err := time.ParseInLocation("2006-01-02", s, time.UTC)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse date %q: %w", s, err)
|
||||
}
|
||||
return &t, nil
|
||||
}
|
||||
Reference in New Issue
Block a user