archivdms/internal/api/document_bulk_export_handlers.go

package api

import (
	"archive/zip"
	"bytes"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"archivdms/internal/audit"
	"archivdms/internal/auth"
	"archivdms/internal/storage"
	"archivdms/internal/userstore"
)

// maxBulkExportDocuments caps how many documents a single bulk export may
// stream. Explicit ID lists beyond this are rejected with 400; a filter
// selection resolving to more documents is likewise rejected so the caller
// narrows the filter instead of silently receiving a truncated archive
// (GoBD-Vollständigkeit: a partial export must never look complete).
const maxBulkExportDocuments = 500

// bulkExportRequest is the POST /api/documents/export body. Either IDs or
// Filter is used — IDs takes precedence when both are present.
//
//	{"ids": [1,2,3]}
//	{"filter": {"doc_type_id": 4, "tag_ids": [7,9],
//	            "document_date_from": "2026-01-01", "document_date_to": "2026-03-31"}}
type bulkExportRequest struct {
	IDs    []int64           `json:"ids"`
	Filter *bulkExportFilter `json:"filter"`
}

// bulkExportFilter mirrors the filter dimensions the list/search endpoints
// already expose (doc type, correspondent, tags, time range). It is applied on
// top of the tenant- and ACL-scoped result of Store.ListDocuments, so no new
// SQL predicate — and no new place a tenant_id filter could be forgotten.
type bulkExportFilter struct {
	DocTypeID        *int64  `json:"doc_type_id"`
	CorrespondentID  *int64  `json:"correspondent_id"`
	TagIDs           []int64 `json:"tag_ids"`
	DocumentDateFrom string  `json:"document_date_from"` // YYYY-MM-DD, inclusive
	DocumentDateTo   string  `json:"document_date_to"`   // YYYY-MM-DD, inclusive
	UploadedFrom     string  `json:"uploaded_from"`      // YYYY-MM-DD, inclusive
	UploadedTo       string  `json:"uploaded_to"`        // YYYY-MM-DD, inclusive (whole day)
}

// bulkExportCSVHeader is the index.csv header. Column names are deliberately
// identical to the metadata.json field names (snake_case) so the later DATEV
// formatter can map from one shared vocabulary.
var bulkExportCSVHeader = []string{
	"document_id", "title", "doc_type", "correspondent",
	"document_date", "tags", "uploaded_at",
}

// handleBulkExportDocuments streams a multi-document ZIP
// (POST /api/documents/export):
//
//	doc-<id>/<title>.<ext>   original WORM file
//	doc-<id>/metadata.json   same shape as the single-document export
//	doc-<id>/ocr_text.txt    only when OCR text exists
//	index.csv                one row per exported document
//	errors.txt               only when documents were skipped
//
// ACL: identical to the single export — tenant scoping plus, for role 'user',
// the per-document document_visibility check. Documents the caller may not see
// (or that fail to read) are skipped and listed in errors.txt rather than
// aborting the whole request.
//
// Audit: exactly ONE EventDocumentBulkExport entry per call, carrying the
// exported/skipped counts.
func (s *Server) handleBulkExportDocuments(w http.ResponseWriter, r *http.Request) {
	sess := sessionFromCtx(r.Context())
	if sess.TenantID == nil {
		writeError(w, http.StatusForbidden, "tenant context required")
		return
	}
	tenantID := *sess.TenantID

	fail := func(status int, msg, detail string) {
		s.audlog.Log(audit.Entry{
			EventType: audit.EventDocumentBulkExport, Username: sess.Username, TenantID: sess.TenantID,
			Success: false, Detail: detail,
		})
		writeError(w, status, msg)
	}

	var req bulkExportRequest
	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
		fail(http.StatusBadRequest, "invalid request body", "decode_failed: "+err.Error())
		return
	}
	if len(req.IDs) == 0 && req.Filter == nil {
		fail(http.StatusBadRequest, "ids oder filter erforderlich", "empty_selection")
		return
	}
	if len(req.IDs) > maxBulkExportDocuments {
		fail(http.StatusBadRequest,
			fmt.Sprintf("maximal %d Dokumente pro Export (angefragt: %d)", maxBulkExportDocuments, len(req.IDs)),
			fmt.Sprintf("too_many_ids: %d", len(req.IDs)))
		return
	}

	// Role 'user' gets the group-resolved ACL applied by the store; domain
	// admins and superadmins see the whole tenant (roles are the outer boundary).
	var aclUserID *int64
	if !auth.HasRole(sess.Role, userstore.RoleDomainAdmin) {
		uid := sess.UserID
		aclUserID = &uid
	}

	// skipped collects IDs that were requested but not exported, with a reason.
	skipped := make([]string, 0, 8)

	// Resolve the selection into a concrete, ordered document list.
	docs := make([]storage.Document, 0, len(req.IDs))
	if len(req.IDs) > 0 {
		for _, id := range req.IDs {
			doc, err := s.store.GetDocument(r.Context(), id, tenantID)
			if err != nil || doc == nil {
				skipped = append(skipped, fmt.Sprintf("%d: nicht gefunden", id))
				continue
			}
			if aclUserID != nil {
				visible, err := s.store.IsDocumentVisible(r.Context(), id, tenantID, *aclUserID)
				if err != nil {
					skipped = append(skipped, fmt.Sprintf("%d: Sichtbarkeitsprüfung fehlgeschlagen", id))
					continue
				}
				if !visible {
					skipped = append(skipped, fmt.Sprintf("%d: nicht sichtbar", id))
					continue
				}
			}
			docs = append(docs, *doc)
		}
	} else {
		all, err := s.store.ListDocuments(r.Context(), tenantID, aclUserID)
		if err != nil {
			fail(http.StatusInternalServerError, "export failed", "list_failed: "+err.Error())
			return
		}
		filtered, err := s.filterBulkExportDocs(r, tenantID, all, req.Filter)
		if err != nil {
			fail(http.StatusBadRequest, err.Error(), "filter_invalid: "+err.Error())
			return
		}
		if len(filtered) > maxBulkExportDocuments {
			fail(http.StatusBadRequest,
				fmt.Sprintf("Filter trifft %d Dokumente, maximal %d pro Export — Filter eingrenzen", len(filtered), maxBulkExportDocuments),
				fmt.Sprintf("filter_too_broad: %d", len(filtered)))
			return
		}
		docs = filtered
	}

	if len(docs) == 0 && len(skipped) == 0 {
		fail(http.StatusNotFound, "keine Dokumente für den Export gefunden", "empty_result")
		return
	}

	ts := time.Now().UTC().Format("20060102-150405")
	w.Header().Set("Content-Type", "application/zip")
	w.Header().Set("Content-Disposition", "attachment; filename=\"export-bulk-"+ts+".zip\"")
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(http.StatusOK)

	zw := zip.NewWriter(w)
	writeEntry := func(name string, rd io.Reader) error {
		entry, err := zw.Create(name)
		if err != nil {
			return err
		}
		_, err = io.Copy(entry, rd)
		return err
	}

	csvBuf := &bytes.Buffer{}
	// UTF-8 BOM so Excel opens the CSV with correct umlauts.
	csvBuf.WriteString("\xef\xbb\xbf")
	cw := csv.NewWriter(csvBuf)
	cw.Comma = ';'
	_ = cw.Write(bulkExportCSVHeader)

	exported := 0
	var streamErr error
	for i := range docs {
		doc := docs[i]
		row, err := s.writeBulkExportDoc(r, writeEntry, sess.Username, tenantID, &doc)
		if err != nil {
			// A single unreadable document must not kill the archive — unless the
			// ZIP writer itself failed, which we detect on Close below.
			s.logger.Warn("bulk export: document skipped", "document_id", doc.ID, "tenant_id", tenantID, "err", err)
			skipped = append(skipped, fmt.Sprintf("%d: %v", doc.ID, err))
			continue
		}
		_ = cw.Write(row)
		exported++
	}
	cw.Flush()

	if streamErr == nil {
		streamErr = writeEntry("index.csv", bytes.NewReader(csvBuf.Bytes()))
	}
	if streamErr == nil && len(skipped) > 0 {
		var b strings.Builder
		b.WriteString("Übersprungene Dokumente (nicht sichtbar, nicht gefunden oder Lesefehler):\n")
		for _, line := range skipped {
			b.WriteString(line)
			b.WriteString("\n")
		}
		streamErr = writeEntry("errors.txt", strings.NewReader(b.String()))
	}
	if closeErr := zw.Close(); streamErr == nil {
		streamErr = closeErr
	}

	detail := fmt.Sprintf("zip_bulk_export: exported=%d skipped=%d", exported, len(skipped))
	if streamErr != nil {
		// Headers are already out — audit the partial export, no HTTP error.
		s.logger.Warn("bulk document export stream failed", "tenant_id", tenantID, "err", streamErr)
		s.audlog.Log(audit.Entry{
			EventType: audit.EventDocumentBulkExport, Username: sess.Username, TenantID: sess.TenantID,
			Success: false, Detail: detail + " stream_failed: " + streamErr.Error(),
		})
		return
	}
	s.audlog.Log(audit.Entry{
		EventType: audit.EventDocumentBulkExport, Username: sess.Username, TenantID: sess.TenantID,
		Success: true, Detail: detail,
	})
}

// writeBulkExportDoc writes the doc-<id>/ folder of one document and returns
// its index.csv row. Any error means "skip this document" — the caller keeps
// the archive going and records the ID in errors.txt.
func (s *Server) writeBulkExportDoc(
	r *http.Request,
	writeEntry func(string, io.Reader) error,
	username string,
	tenantID int64,
	doc *storage.Document,
) ([]string, error) {
	ctx := r.Context()
	prefix := "doc-" + strconv.FormatInt(doc.ID, 10) + "/"

	docTypeName, correspondentName, err := s.store.DocumentTaxonomyNames(ctx, doc.ID, tenantID)
	if err != nil {
		return nil, fmt.Errorf("Taxonomie nicht lesbar: %w", err)
	}
	// Bestandsschutz: fall back to the deprecated free-text columns.
	if docTypeName == "" {
		docTypeName = doc.DocType
	}
	if correspondentName == "" {
		correspondentName = doc.Correspondent
	}

	tagEntities, err := s.store.ListDocumentTags(ctx, doc.ID, tenantID)
	if err != nil {
		return nil, fmt.Errorf("Tags nicht lesbar: %w", err)
	}
	tags := make([]string, 0, len(tagEntities))
	for _, t := range tagEntities {
		tags = append(tags, t.Name)
	}

	fieldValues, err := s.store.ListDocumentFieldValues(ctx, doc.ID, tenantID)
	if err != nil {
		return nil, fmt.Errorf("Zusatzfelder nicht lesbar: %w", err)
	}

	f, err := os.Open(doc.StoragePath)
	if err != nil {
		return nil, fmt.Errorf("Datei nicht lesbar: %w", err)
	}
	defer f.Close()

	meta := documentExportMetadata{
		DocumentID:        doc.ID,
		TenantID:          doc.TenantID,
		Title:             doc.Title,
		DocType:           docTypeName,
		Correspondent:     correspondentName,
		Tags:              tags,
		DocumentDateScore: doc.DocumentDateScore,
		UploadedAt:        doc.CreatedAt,
		UpdatedAt:         doc.UpdatedAt,
		CreatedBy:         s.exportCreatorName(doc),
		ContentHash:       doc.ContentHash,
		OriginalFilename:  filepath.Base(doc.StoragePath),
		CustomFields:      exportCustomFields(fieldValues),
		ExportedAt:        time.Now().UTC(),
		ExportedBy:        username,
	}
	docDate := ""
	if doc.DocumentDate != nil {
		docDate = doc.DocumentDate.Format("2006-01-02")
		meta.DocumentDate = &docDate
	}
	if doc.RetainUntil != nil {
		d := doc.RetainUntil.Format("2006-01-02")
		meta.RetainUntil = &d
	}
	metaJSON, err := json.MarshalIndent(meta, "", "  ")
	if err != nil {
		return nil, fmt.Errorf("Metadaten nicht serialisierbar: %w", err)
	}

	ext := filepath.Ext(doc.StoragePath)
	if err := writeEntry(prefix+safeDownloadName(doc.Title, ext), f); err != nil {
		return nil, fmt.Errorf("ZIP-Eintrag fehlgeschlagen: %w", err)
	}
	if err := writeEntry(prefix+"metadata.json", bytes.NewReader(metaJSON)); err != nil {
		return nil, fmt.Errorf("ZIP-Eintrag fehlgeschlagen: %w", err)
	}
	if doc.OCRText != "" {
		if err := writeEntry(prefix+"ocr_text.txt", strings.NewReader(doc.OCRText)); err != nil {
			return nil, fmt.Errorf("ZIP-Eintrag fehlgeschlagen: %w", err)
		}
	}

	return []string{
		strconv.FormatInt(doc.ID, 10),
		doc.Title,
		docTypeName,
		correspondentName,
		docDate,
		strings.Join(tags, ", "),
		doc.CreatedAt.UTC().Format(time.RFC3339),
	}, nil
}

// filterBulkExportDocs narrows an already tenant- and ACL-scoped document list
// by the requested filter. Tag filtering needs a per-document lookup, so it is
// applied last, after the cheap in-memory predicates.
func (s *Server) filterBulkExportDocs(r *http.Request, tenantID int64, docs []storage.Document, f *bulkExportFilter) ([]storage.Document, error) {
	docFrom, err := parseBulkExportDate(f.DocumentDateFrom)
	if err != nil {
		return nil, fmt.Errorf("ungültiges document_date_from (erwartet YYYY-MM-DD)")
	}
	docTo, err := parseBulkExportDate(f.DocumentDateTo)
	if err != nil {
		return nil, fmt.Errorf("ungültiges document_date_to (erwartet YYYY-MM-DD)")
	}
	upFrom, err := parseBulkExportDate(f.UploadedFrom)
	if err != nil {
		return nil, fmt.Errorf("ungültiges uploaded_from (erwartet YYYY-MM-DD)")
	}
	upTo, err := parseBulkExportDate(f.UploadedTo)
	if err != nil {
		return nil, fmt.Errorf("ungültiges uploaded_to (erwartet YYYY-MM-DD)")
	}

	out := make([]storage.Document, 0, len(docs))
	for i := range docs {
		d := docs[i]
		if f.DocTypeID != nil && (d.DocTypeID == nil || *d.DocTypeID != *f.DocTypeID) {
			continue
		}
		if f.CorrespondentID != nil && (d.CorrespondentID == nil || *d.CorrespondentID != *f.CorrespondentID) {
			continue
		}
		if docFrom != nil || docTo != nil {
			if d.DocumentDate == nil {
				continue
			}
			day := d.DocumentDate.UTC().Truncate(24 * time.Hour)
			if docFrom != nil && day.Before(*docFrom) {
				continue
			}
			if docTo != nil && day.After(*docTo) {
				continue
			}
		}
		if upFrom != nil && d.CreatedAt.UTC().Before(*upFrom) {
			continue
		}
		if upTo != nil && d.CreatedAt.UTC().After(upTo.Add(24*time.Hour-time.Nanosecond)) {
			continue
		}
		out = append(out, d)
	}

	if len(f.TagIDs) == 0 {
		return out, nil
	}
	want := make(map[int64]struct{}, len(f.TagIDs))
	for _, id := range f.TagIDs {
		want[id] = struct{}{}
	}
	tagged := make([]storage.Document, 0, len(out))
	for i := range out {
		tags, err := s.store.ListDocumentTags(r.Context(), out[i].ID, tenantID)
		if err != nil {
			return nil, fmt.Errorf("Tag-Filter fehlgeschlagen")
		}
		for _, t := range tags {
			if _, ok := want[t.ID]; ok {
				tagged = append(tagged, out[i])
				break
			}
		}
	}
	return tagged, nil
}

// parseBulkExportDate parses an optional YYYY-MM-DD filter bound (UTC).
func parseBulkExportDate(s string) (*time.Time, error) {
	s = strings.TrimSpace(s)
	if s == "" {
		return nil, nil
	}
	t, err := time.ParseInLocation("2006-01-02", s, time.UTC)
	if err != nil {
		return nil, fmt.Errorf("parse date %q: %w", s, err)
	}
	return &t, nil
}