From 21e15d71b898011e1bdbf83856e9bd44787615f0 Mon Sep 17 00:00:00 2001 From: patrick Date: Tue, 3 Nov 2026 23:41:18 +0100 Subject: [PATCH] dms/backup-tool: Quelle und Ziel waehlbar lokal oder S3 Backup und Restore nutzen NewDriverFromEnv mit den Praefixen NEXARCH_DMS_BACKUP_ und NEXARCH_DMS_RESTORE_. Ohne STORAGE_DRIVER bleibt es beim lokalen Verzeichnis. Der Drill-Test laeuft mit TEST_S3_SRC_BUCKET und TEST_S3_BUCKET gegen S3. Co-Authored-By: Claude Sonnet 5.5 --- dms/README.md | 2 +- dms/cmd/backup-tool/backup.go | 6 +++++- dms/cmd/backup-tool/drill_test.go | 30 +++++++++++++++++++++++++++--- dms/cmd/backup-tool/main.go | 11 +++++++++-- dms/cmd/backup-tool/restore.go | 6 +++++- 5 files changed, 47 insertions(+), 8 deletions(-) diff --git a/dms/README.md b/dms/README.md index 42896720..9e3072fc 100644 --- a/dms/README.md +++ b/dms/README.md @@ -31,7 +31,7 @@ registrierte Rückruf-URL (`NEXARCH_DMS_APP_RETENTION_CALLBACK_URL`) muss den Pr ## Objekt-Speicher -`NEXARCH_DMS_APP_STORAGE_DRIVER` und `NEXARCH_DMS_WORKER_STORAGE_DRIVER` wählen den Treiber: `local` (Standard, braucht `…_STORAGE_DIR`) oder `s3`. Für `s3` sind `…_S3_BUCKET`, `…_S3_ACCESS_KEY_ID` und `…_S3_SECRET_ACCESS_KEY` Pflicht; optional `…_S3_REGION` (Standard `us-east-1`), `…_S3_ENDPOINT_URL` (MinIO, Ceph) und `…_S3_PATH_STYLE=1`. App und Worker müssen denselben Bucket nutzen. +`NEXARCH_DMS_APP_STORAGE_DRIVER` und `NEXARCH_DMS_WORKER_STORAGE_DRIVER` wählen den Treiber: `local` (Standard, braucht `…_STORAGE_DIR`) oder `s3`. Für `s3` sind `…_S3_BUCKET`, `…_S3_ACCESS_KEY_ID` und `…_S3_SECRET_ACCESS_KEY` Pflicht; optional `…_S3_REGION` (Standard `us-east-1`), `…_S3_ENDPOINT_URL` (MinIO, Ceph) und `…_S3_PATH_STYLE=1`. App und Worker müssen denselben Bucket nutzen. Das `backup-tool` liest dieselben Variablen mit den Präfixen `NEXARCH_DMS_BACKUP_` (Quelle) und `NEXARCH_DMS_RESTORE_` (Ziel), auch mit `…_STORAGE_DRIVER=s3`. ## Struktur diff --git a/dms/cmd/backup-tool/backup.go b/dms/cmd/backup-tool/backup.go index 73cac158..1023a52b 100644 --- a/dms/cmd/backup-tool/backup.go +++ b/dms/cmd/backup-tool/backup.go @@ -14,6 +14,7 @@ import ( type backupConfig struct { DSN string StorageDir string + Driver storage.Driver TenantSlug string OutDir string PgDumpPath string @@ -45,7 +46,10 @@ func runBackup(ctx context.Context, cfg backupConfig) error { } defer pool.Close() - driver := storage.NewLocalDriver(cfg.StorageDir, []byte("backup-tool-unused"), "", 0) + driver := cfg.Driver + if driver == nil { + driver = storage.NewLocalDriver(cfg.StorageDir, []byte("backup-tool-unused"), "", 0) + } rows, err := pool.Query(ctx, ` SELECT r.id, r.storage_key, r.size_bytes diff --git a/dms/cmd/backup-tool/drill_test.go b/dms/cmd/backup-tool/drill_test.go index eefe2a43..1644c161 100644 --- a/dms/cmd/backup-tool/drill_test.go +++ b/dms/cmd/backup-tool/drill_test.go @@ -32,6 +32,27 @@ const drillTenantSlug = "fdn27drill" // des Tickets): ein echter, unabhaengiger Quell- und Zielbestand auf dem // Entwicklungshost, kein simulierter Lauf. Er belegt AK1 bis AK3 in einem // zusammenhaengenden Durchlauf und misst RTO/RPO real. +// drillDriver liefert den lokalen Treiber, oder einen S3-Treiber, wenn die +// Umgebungsvariable bucketEnv einen Bucket nennt (TEST_S3_ENDPOINT usw. wie +// in dms/internal/storage). +func drillDriver(t *testing.T, bucketEnv, dir string) storage.Driver { + t.Helper() + bucket := os.Getenv(bucketEnv) + if bucket == "" { + return storage.NewLocalDriver(dir, []byte("secret"), "https://files.example.test", 0) + } + region := os.Getenv("TEST_S3_REGION") + if region == "" { + region = "us-east-1" + } + d, err := storage.NewS3Driver(context.Background(), bucket, region, os.Getenv("TEST_S3_ENDPOINT"), + os.Getenv("TEST_S3_ACCESS_KEY_ID"), os.Getenv("TEST_S3_SECRET_ACCESS_KEY"), true, 0) + if err != nil { + t.Fatalf("s3-treiber: %v", err) + } + return d +} + func TestBackupRestoreDrill_RealRoundTrip(t *testing.T) { dsn := os.Getenv("TEST_TENANT_DSN") if dsn == "" { @@ -84,7 +105,8 @@ func TestBackupRestoreDrill_RealRoundTrip(t *testing.T) { } srcStorageDir := t.TempDir() - srcStorageSvc := storage.NewService(storage.NewLocalDriver(srcStorageDir, []byte("secret"), "https://files.example.test", 0), drillNoopUsage{}, drillTenantSlug) + srcDriver := drillDriver(t, "TEST_S3_SRC_BUCKET", srcStorageDir) + srcStorageSvc := storage.NewService(srcDriver, drillNoopUsage{}, drillTenantSlug) uploadSvc := upload.NewService(srcPool, upload.NewSessionStore(srcPool), upload.NewStaging(t.TempDir()), srcStorageSvc, crypt, upload.NewValidator(10*1024*1024, nil), drillTenantSlug) const wantDocs = 3 @@ -108,6 +130,7 @@ func TestBackupRestoreDrill_RealRoundTrip(t *testing.T) { if err := runBackup(ctx, backupConfig{ DSN: dsn, StorageDir: srcStorageDir, + Driver: srcDriver, TenantSlug: drillTenantSlug, OutDir: backupDir, PgSchema: srcSchema, @@ -128,11 +151,13 @@ func TestBackupRestoreDrill_RealRoundTrip(t *testing.T) { dstSchema := fmt.Sprintf("fdn27dst_%d", time.Now().UnixNano()) t.Cleanup(func() { _, _ = admin.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+dstSchema+" CASCADE") }) dstStorageDir := t.TempDir() + dstDriver := drillDriver(t, "TEST_S3_BUCKET", dstStorageDir) restoreStart := time.Now() if err := runRestore(ctx, restoreConfig{ DSN: dsn, StorageDir: dstStorageDir, + Driver: dstDriver, InDir: backupDir, TargetSchema: dstSchema, Crypto: crypt, @@ -162,10 +187,9 @@ func TestBackupRestoreDrill_RealRoundTrip(t *testing.T) { // -- FDN-23-Integritaetsjob auf dem wiederhergestellten Bestand: muss // "ok" liefern, sonst waere der Restore nur scheinbar vollstaendig -- - dstStorageSvc := storage.NewLocalDriver(dstStorageDir, []byte("secret"), "https://files.example.test", 0) verifier := &integrity.Verifier{ Pool: dstPool, - Storage: storage.NewService(dstStorageSvc, drillNoopUsage{}, drillTenantSlug), + Storage: storage.NewService(dstDriver, drillNoopUsage{}, drillTenantSlug), Crypto: crypt, TenantSlug: drillTenantSlug, Config: integrity.Config{Mode: integrity.ModeFull, Batch: 100}, diff --git a/dms/cmd/backup-tool/main.go b/dms/cmd/backup-tool/main.go index 08604a3b..3035d7f2 100644 --- a/dms/cmd/backup-tool/main.go +++ b/dms/cmd/backup-tool/main.go @@ -7,6 +7,7 @@ import ( "os" dmscrypto "gitea.perlbach24.de/scripte/nexarch/dms/internal/crypto" + "gitea.perlbach24.de/scripte/nexarch/dms/internal/storage" ) func main() { @@ -18,9 +19,15 @@ func main() { var err error switch os.Args[1] { case "backup": - err = runBackup(ctx, backupConfigFromEnv(os.Args[2])) + cfg := backupConfigFromEnv(os.Args[2]) + if cfg.Driver, err = storage.NewDriverFromEnv(ctx, "NEXARCH_DMS_BACKUP"); err == nil { + err = runBackup(ctx, cfg) + } case "restore": - err = runRestore(ctx, restoreConfigFromEnv(os.Args[2])) + cfg := restoreConfigFromEnv(os.Args[2]) + if cfg.Driver, err = storage.NewDriverFromEnv(ctx, "NEXARCH_DMS_RESTORE"); err == nil { + err = runRestore(ctx, cfg) + } default: usage() os.Exit(2) diff --git a/dms/cmd/backup-tool/restore.go b/dms/cmd/backup-tool/restore.go index f6218a03..67d55681 100644 --- a/dms/cmd/backup-tool/restore.go +++ b/dms/cmd/backup-tool/restore.go @@ -23,6 +23,7 @@ var errTargetNotEmpty = errors.New("restore: zieldatenbank ist nicht leer (--for type restoreConfig struct { DSN string StorageDir string + Driver storage.Driver InDir string PgRestorePath string PsqlPath string @@ -69,7 +70,10 @@ func runRestore(ctx context.Context, cfg restoreConfig) error { } } - driver := storage.NewLocalDriver(cfg.StorageDir, []byte("backup-tool-unused"), "", 0) + driver := cfg.Driver + if driver == nil { + driver = storage.NewLocalDriver(cfg.StorageDir, []byte("backup-tool-unused"), "", 0) + } for _, obj := range m.Objects { if err := restoreObject(ctx, driver, cfg.InDir, obj); err != nil { return fmt.Errorf("restore: objekt %q (revision %s): %w", obj.StorageKey, obj.RevisionID, err) -- 2.47.3