Fortsetzbarer Server-seitiger Upload: upload_sessions (bytes_received, GREATEST-Update verhindert Rueckschritt bei erneut zugestellten Chunks), Staging via os.File.WriteAt (beliebige Chunk-Reihenfolge/-Wiederholung), MIME-/Groessen-Validierung vor jedem Byte. Complete() liest Klartext einmal via io.TeeReader fuer SHA-256 UND Verschluesselung gleichzeitig (Reihenfolge Hash->verschluesseln->ablegen eingehalten), legt Dokument+Revision transaktional an (neue Spalte file_revisions.wrapped_dek fuer FDN-09). Auf 192.168.1.131 verifiziert: Resume nach simuliertem Abbruch bei 50% liefert identische Endpruefsumme, 20 parallele Uploads ohne Kollision/ Datenverlust, Pruefsumme entspricht exakt dem Klartext, transaktionale Dokument+Revision-Anlage bestaetigt. Reale Skalierungs-Einschraenkung gefunden: echter 1-GiB-Durchlauf endete mit OOM (4GB-RAM-Testhost ohne Swap, FDN-03/FDN-09 puffern vollstaendig im Speicher statt zu streamen). Auf 300 MiB reduziert vollstaendig verifiziert (Upload/Verschluesselung/Ablage/Checksumme/Entschluesselungs-Round-Trip alles exakt) - Ressourcen-, keine Korrektheitsfrage, dokumentiert als Folgeticket-Kandidat statt stillschweigend uebergangen. Siehe dms/docs/DOC-01-PRUEFPROTOKOLL.md fuer alle Pruefungsergebnisse. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HhgFcLS8tYMhDJpP74C6AQ
300 lines
11 KiB
Go
300 lines
11 KiB
Go
package upload
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
|
|
dmscrypto "gitea.perlbach24.de/scripte/nexarch/dms/internal/crypto"
|
|
"gitea.perlbach24.de/scripte/nexarch/dms/internal/storage"
|
|
)
|
|
|
|
// fakeKEKProvider liefert einen fest hinterlegten Tenant-KEK — dieselbe
|
|
// Fixture wie in internal/crypto, hier lokal dupliziert, da Testhilfen
|
|
// nicht paketuebergreifend exportiert sind.
|
|
type fakeKEKProvider struct{ kek []byte }
|
|
|
|
func (f *fakeKEKProvider) TenantKEK(ctx context.Context, tenantSlug string) ([]byte, error) {
|
|
return f.kek, nil
|
|
}
|
|
|
|
func setupServiceTest(t *testing.T) (*Service, *pgxpool.Pool, string) {
|
|
t.Helper()
|
|
dsn := os.Getenv("TEST_TENANT_DSN")
|
|
if dsn == "" {
|
|
t.Skip("TEST_TENANT_DSN nicht gesetzt, Integrationstest uebersprungen")
|
|
}
|
|
ctx := context.Background()
|
|
|
|
pool, err := pgxpool.New(ctx, dsn)
|
|
if err != nil {
|
|
t.Fatalf("pool: %v", err)
|
|
}
|
|
t.Cleanup(func() { pool.Close() })
|
|
|
|
if _, err := pool.Exec(ctx, `
|
|
CREATE EXTENSION IF NOT EXISTS pgcrypto;
|
|
CREATE TABLE IF NOT EXISTS users (
|
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(), email TEXT NOT NULL UNIQUE, name TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'active', created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
CREATE TABLE IF NOT EXISTS folders (
|
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(), parent_folder_id UUID REFERENCES folders(id) ON DELETE CASCADE,
|
|
name TEXT NOT NULL, created_by UUID NOT NULL REFERENCES users(id),
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
CREATE TABLE IF NOT EXISTS documents (
|
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(), folder_id UUID REFERENCES folders(id) ON DELETE SET NULL,
|
|
title TEXT NOT NULL, current_revision_id UUID, created_by UUID NOT NULL REFERENCES users(id),
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), deleted_at TIMESTAMPTZ
|
|
);
|
|
CREATE TABLE IF NOT EXISTS file_revisions (
|
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(), document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
|
revision_number INT NOT NULL, storage_key TEXT NOT NULL, checksum_sha256 TEXT NOT NULL,
|
|
size_bytes BIGINT NOT NULL, mime_type TEXT NOT NULL, created_by UUID NOT NULL REFERENCES users(id),
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(), wrapped_dek BYTEA,
|
|
UNIQUE (document_id, revision_number)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS upload_sessions (
|
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(), filename TEXT NOT NULL, mime_type TEXT NOT NULL,
|
|
total_size BIGINT NOT NULL CHECK (total_size > 0), bytes_received BIGINT NOT NULL DEFAULT 0 CHECK (bytes_received >= 0),
|
|
folder_id UUID REFERENCES folders(id) ON DELETE SET NULL, created_by UUID NOT NULL REFERENCES users(id),
|
|
status TEXT NOT NULL DEFAULT 'uploading' CHECK (status IN ('uploading','completed','aborted')),
|
|
document_id UUID REFERENCES documents(id),
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
`); err != nil {
|
|
t.Fatalf("schema: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
ctx := context.Background()
|
|
_, _ = pool.Exec(ctx, `TRUNCATE upload_sessions, file_revisions, documents, folders, users CASCADE`)
|
|
})
|
|
|
|
var userID string
|
|
if err := pool.QueryRow(ctx, `
|
|
INSERT INTO users (email, name) VALUES ($1, 'Test-Benutzer') RETURNING id
|
|
`, fmt.Sprintf("upload-test-%d@example.test", time.Now().UnixNano())).Scan(&userID); err != nil {
|
|
t.Fatalf("testbenutzer anlegen: %v", err)
|
|
}
|
|
|
|
sessions := NewSessionStore(pool)
|
|
staging := NewStaging(t.TempDir())
|
|
storageSvc := storage.NewService(storage.NewLocalDriver(t.TempDir(), []byte("secret"), "https://files.example.test"), noopUsageReporter{}, "acme")
|
|
cryptoSvc := dmscrypto.NewService(&fakeKEKProvider{kek: bytes.Repeat([]byte{0x11}, dmscrypto.KEKSize)})
|
|
validator := NewValidator(10*1024*1024, []string{"application/pdf", "text/plain"})
|
|
|
|
svc := NewService(pool, sessions, staging, storageSvc, cryptoSvc, validator, "acme")
|
|
return svc, pool, userID
|
|
}
|
|
|
|
// noopUsageReporter ersetzt den echten HTTPUsageReporter aus FDN-03 fuer
|
|
// diese Tests — Nutzungsmeldung ist bereits in FDN-03 eigenstaendig
|
|
// getestet, hier geht es nur um den Upload-Pfad selbst.
|
|
type noopUsageReporter struct{}
|
|
|
|
func (noopUsageReporter) Report(ctx context.Context, tenantSlug, metric string, delta int64) error {
|
|
return nil
|
|
}
|
|
|
|
// TestUploadChunk_DisallowedMimeTypeRejected ist Akzeptanzkriterium 2.
|
|
func TestUploadChunk_DisallowedMimeTypeRejected(t *testing.T) {
|
|
svc, _, userID := setupServiceTest(t)
|
|
ctx := context.Background()
|
|
|
|
_, err := svc.StartUpload(ctx, "schadcode.exe", "application/x-msdownload", 100, nil, userID)
|
|
if !errors.Is(err, ErrDisallowedMimeType) {
|
|
t.Fatalf("erwartet ErrDisallowedMimeType, habe %v", err)
|
|
}
|
|
}
|
|
|
|
// TestUploadChunk_OversizedRejected prueft die Groessenpruefung
|
|
// (Schutz vor Speicherbomben).
|
|
func TestUploadChunk_OversizedRejected(t *testing.T) {
|
|
svc, _, userID := setupServiceTest(t)
|
|
ctx := context.Background()
|
|
|
|
_, err := svc.StartUpload(ctx, "riesig.pdf", "application/pdf", 100*1024*1024, nil, userID)
|
|
if !errors.Is(err, ErrFileTooLarge) {
|
|
t.Fatalf("erwartet ErrFileTooLarge, habe %v", err)
|
|
}
|
|
}
|
|
|
|
// TestCompleteUpload_CreatesDocumentAndRevisionTransactionally ist
|
|
// Akzeptanzkriterium 3 UND 4 (Pruefsumme).
|
|
func TestCompleteUpload_CreatesDocumentAndRevisionTransactionally(t *testing.T) {
|
|
svc, pool, userID := setupServiceTest(t)
|
|
ctx := context.Background()
|
|
|
|
content := []byte("Rechnung 2026-0001 — Testinhalt fuer DOC-01")
|
|
sum := sha256.Sum256(content)
|
|
wantChecksum := hex.EncodeToString(sum[:])
|
|
|
|
sessionID, err := svc.StartUpload(ctx, "rechnung.pdf", "application/pdf", int64(len(content)), nil, userID)
|
|
if err != nil {
|
|
t.Fatalf("startupload: %v", err)
|
|
}
|
|
if _, err := svc.UploadChunk(ctx, sessionID, 0, bytes.NewReader(content)); err != nil {
|
|
t.Fatalf("uploadchunk: %v", err)
|
|
}
|
|
|
|
documentID, revisionID, err := svc.Complete(ctx, sessionID)
|
|
if err != nil {
|
|
t.Fatalf("complete: %v", err)
|
|
}
|
|
if documentID == "" || revisionID == "" {
|
|
t.Fatal("erwartet nicht-leere document/revision-ids")
|
|
}
|
|
|
|
var title string
|
|
var currentRevisionID *string
|
|
if err := pool.QueryRow(ctx, `SELECT title, current_revision_id FROM documents WHERE id = $1`, documentID).Scan(&title, ¤tRevisionID); err != nil {
|
|
t.Fatalf("dokument lesen: %v", err)
|
|
}
|
|
if title != "rechnung.pdf" {
|
|
t.Fatalf("titel = %q, want %q", title, "rechnung.pdf")
|
|
}
|
|
if currentRevisionID == nil || *currentRevisionID != revisionID {
|
|
t.Fatalf("current_revision_id = %v, want %q", currentRevisionID, revisionID)
|
|
}
|
|
|
|
var checksum string
|
|
var wrappedDEK []byte
|
|
if err := pool.QueryRow(ctx, `SELECT checksum_sha256, wrapped_dek FROM file_revisions WHERE id = $1`, revisionID).Scan(&checksum, &wrappedDEK); err != nil {
|
|
t.Fatalf("revision lesen: %v", err)
|
|
}
|
|
if checksum != wantChecksum {
|
|
t.Fatalf("checksum_sha256 = %q, want %q (sha256 des klartexts)", checksum, wantChecksum)
|
|
}
|
|
if len(wrappedDEK) == 0 {
|
|
t.Fatal("erwartet nicht-leeren wrapped_dek (objekt wurde verschluesselt)")
|
|
}
|
|
}
|
|
|
|
// TestUploadChunk_ResumeAfterAbortProducesIdenticalChecksum ist
|
|
// Akzeptanzkriterium 1 / Pruefung 2: Abbruch bei 50% und Fortsetzung ergibt
|
|
// identische Pruefsumme.
|
|
func TestUploadChunk_ResumeAfterAbortProducesIdenticalChecksum(t *testing.T) {
|
|
svc, _, userID := setupServiceTest(t)
|
|
ctx := context.Background()
|
|
|
|
content := make([]byte, 200*1024) // 200 KiB
|
|
if _, err := rand.Read(content); err != nil {
|
|
t.Fatalf("zufallsinhalt erzeugen: %v", err)
|
|
}
|
|
sum := sha256.Sum256(content)
|
|
wantChecksum := hex.EncodeToString(sum[:])
|
|
|
|
sessionID, err := svc.StartUpload(ctx, "grosse-datei.pdf", "application/pdf", int64(len(content)), nil, userID)
|
|
if err != nil {
|
|
t.Fatalf("startupload: %v", err)
|
|
}
|
|
|
|
half := len(content) / 2
|
|
if _, err := svc.UploadChunk(ctx, sessionID, 0, bytes.NewReader(content[:half])); err != nil {
|
|
t.Fatalf("uploadchunk (erste haelfte): %v", err)
|
|
}
|
|
|
|
// Simulierter Verbindungsabbruch: Sitzung abfragen wie ein Client, der
|
|
// nach dem Abbruch neu verbindet und wissen will, wo er stand.
|
|
status, err := svc.Status(ctx, sessionID)
|
|
if err != nil {
|
|
t.Fatalf("status: %v", err)
|
|
}
|
|
if status.BytesReceived != int64(half) {
|
|
t.Fatalf("bytes_received nach abbruch = %d, want %d", status.BytesReceived, half)
|
|
}
|
|
|
|
// Fortsetzung GENAU ab dem zuletzt bestaetigten Offset.
|
|
if _, err := svc.UploadChunk(ctx, sessionID, int64(half), bytes.NewReader(content[half:])); err != nil {
|
|
t.Fatalf("uploadchunk (fortsetzung): %v", err)
|
|
}
|
|
|
|
_, revisionID, err := svc.Complete(ctx, sessionID)
|
|
if err != nil {
|
|
t.Fatalf("complete: %v", err)
|
|
}
|
|
|
|
got := checksumOf(t, svc, revisionID)
|
|
if got != wantChecksum {
|
|
t.Fatalf("checksum nach fortgesetztem upload = %q, want %q", got, wantChecksum)
|
|
}
|
|
}
|
|
|
|
func checksumOf(t *testing.T, svc *Service, revisionID string) string {
|
|
t.Helper()
|
|
var checksum string
|
|
if err := svc.pool.QueryRow(context.Background(), `SELECT checksum_sha256 FROM file_revisions WHERE id = $1`, revisionID).Scan(&checksum); err != nil {
|
|
t.Fatalf("checksum lesen: %v", err)
|
|
}
|
|
return checksum
|
|
}
|
|
|
|
// TestParallelUploads_NoDataLoss ist Pruefung 3: Parallel-Upload von 20
|
|
// Dateien ohne Datenverlust.
|
|
func TestParallelUploads_NoDataLoss(t *testing.T) {
|
|
svc, _, userID := setupServiceTest(t)
|
|
ctx := context.Background()
|
|
const n = 20
|
|
|
|
type result struct {
|
|
documentID string
|
|
checksum string
|
|
}
|
|
results := make([]result, n)
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < n; i++ {
|
|
wg.Add(1)
|
|
go func(idx int) {
|
|
defer wg.Done()
|
|
content := []byte(fmt.Sprintf("paralleler inhalt nummer %d, eindeutig genug fuer eigenen hash", idx))
|
|
sum := sha256.Sum256(content)
|
|
wantChecksum := hex.EncodeToString(sum[:])
|
|
|
|
sessionID, err := svc.StartUpload(ctx, fmt.Sprintf("datei-%d.pdf", idx), "application/pdf", int64(len(content)), nil, userID)
|
|
if err != nil {
|
|
t.Errorf("startupload %d: %v", idx, err)
|
|
return
|
|
}
|
|
if _, err := svc.UploadChunk(ctx, sessionID, 0, bytes.NewReader(content)); err != nil {
|
|
t.Errorf("uploadchunk %d: %v", idx, err)
|
|
return
|
|
}
|
|
docID, revID, err := svc.Complete(ctx, sessionID)
|
|
if err != nil {
|
|
t.Errorf("complete %d: %v", idx, err)
|
|
return
|
|
}
|
|
results[idx] = result{documentID: docID, checksum: checksumOf(t, svc, revID)}
|
|
if results[idx].checksum != wantChecksum {
|
|
t.Errorf("upload %d: checksum = %q, want %q", idx, results[idx].checksum, wantChecksum)
|
|
}
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
|
|
seen := map[string]bool{}
|
|
for i, r := range results {
|
|
if r.documentID == "" {
|
|
t.Fatalf("upload %d lieferte keine document-id (fehlgeschlagen)", i)
|
|
}
|
|
if seen[r.documentID] {
|
|
t.Fatalf("document-id %q doppelt vergeben - datenverlust/kollision", r.documentID)
|
|
}
|
|
seen[r.documentID] = true
|
|
}
|
|
if len(seen) != n {
|
|
t.Fatalf("erwartet %d eindeutige dokumente, habe %d", n, len(seen))
|
|
}
|
|
}
|