scripte/nexarch
1
0 Fork 0
Code Issues 1 Pull Requests 21 Actions 99 Packages Projects Releases Wiki Activity

dms/backup-tool: Quelle und Ziel lokal oder S3 #432

Open
patrick wants to merge 1 commits from feat/dms-backup-s3 into feat/dms-s3-verdrahten
pull from: feat/dms-backup-s3
merge into: scripte:feat/dms-s3-verdrahten
Conversation 0 Commits 1 Files Changed 5
+47 -8
5 changed files with 47 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

dms/README.md
+1 -1
View File
@@ -31,7 +31,7 @@ registrierte Rückruf-URL (`NEXARCH_DMS_APP_RETENTION_CALLBACK_URL`) muss den Pr
## Objekt-Speicher
`NEXARCH_DMS_APP_STORAGE_DRIVER` und `NEXARCH_DMS_WORKER_STORAGE_DRIVER` wählen den Treiber: `local` (Standard, braucht `…_STORAGE_DIR`) oder `s3`. Für `s3` sind `…_S3_BUCKET`, `…_S3_ACCESS_KEY_ID` und `…_S3_SECRET_ACCESS_KEY` Pflicht; optional `…_S3_REGION` (Standard `us-east-1`), `…_S3_ENDPOINT_URL` (MinIO, Ceph) und `…_S3_PATH_STYLE=1`. App und Worker müssen denselben Bucket nutzen.
`NEXARCH_DMS_APP_STORAGE_DRIVER` und `NEXARCH_DMS_WORKER_STORAGE_DRIVER` wählen den Treiber: `local` (Standard, braucht `…_STORAGE_DIR`) oder `s3`. Für `s3` sind `…_S3_BUCKET`, `…_S3_ACCESS_KEY_ID` und `…_S3_SECRET_ACCESS_KEY` Pflicht; optional `…_S3_REGION` (Standard `us-east-1`), `…_S3_ENDPOINT_URL` (MinIO, Ceph) und `…_S3_PATH_STYLE=1`. App und Worker müssen denselben Bucket nutzen. Das `backup-tool` liest dieselben Variablen mit den Präfixen `NEXARCH_DMS_BACKUP_` (Quelle) und `NEXARCH_DMS_RESTORE_` (Ziel), auch mit `…_STORAGE_DRIVER=s3`.
## Struktur
dms/cmd/backup-tool/backup.go
+5 -1
View File
@@ -14,6 +14,7 @@ import (
type backupConfig struct {
DSN string
StorageDir string
Driver storage.Driver
TenantSlug string
OutDir string
PgDumpPath string
@@ -45,7 +46,10 @@ func runBackup(ctx context.Context, cfg backupConfig) error {
}
defer pool.Close()
driver := storage.NewLocalDriver(cfg.StorageDir, []byte("backup-tool-unused"), "", 0)
driver := cfg.Driver
if driver == nil {
driver = storage.NewLocalDriver(cfg.StorageDir, []byte("backup-tool-unused"), "", 0)
}
rows, err := pool.Query(ctx, `
SELECT r.id, r.storage_key, r.size_bytes
dms/cmd/backup-tool/drill_test.go
+27 -3
View File
@@ -32,6 +32,27 @@ const drillTenantSlug = "fdn27drill"
// des Tickets): ein echter, unabhaengiger Quell- und Zielbestand auf dem
// Entwicklungshost, kein simulierter Lauf. Er belegt AK1 bis AK3 in einem
// zusammenhaengenden Durchlauf und misst RTO/RPO real.
// drillDriver liefert den lokalen Treiber, oder einen S3-Treiber, wenn die
// Umgebungsvariable bucketEnv einen Bucket nennt (TEST_S3_ENDPOINT usw. wie
// in dms/internal/storage).
func drillDriver(t *testing.T, bucketEnv, dir string) storage.Driver {
t.Helper()
bucket := os.Getenv(bucketEnv)
if bucket == "" {
return storage.NewLocalDriver(dir, []byte("secret"), "https://files.example.test", 0)
}
region := os.Getenv("TEST_S3_REGION")
if region == "" {
region = "us-east-1"
}
d, err := storage.NewS3Driver(context.Background(), bucket, region, os.Getenv("TEST_S3_ENDPOINT"),
os.Getenv("TEST_S3_ACCESS_KEY_ID"), os.Getenv("TEST_S3_SECRET_ACCESS_KEY"), true, 0)
if err != nil {
t.Fatalf("s3-treiber: %v", err)
}
return d
}
func TestBackupRestoreDrill_RealRoundTrip(t *testing.T) {
dsn := os.Getenv("TEST_TENANT_DSN")
if dsn == "" {
@@ -84,7 +105,8 @@ func TestBackupRestoreDrill_RealRoundTrip(t *testing.T) {
}
srcStorageDir := t.TempDir()
srcStorageSvc := storage.NewService(storage.NewLocalDriver(srcStorageDir, []byte("secret"), "https://files.example.test", 0), drillNoopUsage{}, drillTenantSlug)
srcDriver := drillDriver(t, "TEST_S3_SRC_BUCKET", srcStorageDir)
srcStorageSvc := storage.NewService(srcDriver, drillNoopUsage{}, drillTenantSlug)
uploadSvc := upload.NewService(srcPool, upload.NewSessionStore(srcPool), upload.NewStaging(t.TempDir()), srcStorageSvc, crypt, upload.NewValidator(10*1024*1024, nil), drillTenantSlug)
const wantDocs = 3
@@ -108,6 +130,7 @@ func TestBackupRestoreDrill_RealRoundTrip(t *testing.T) {
if err := runBackup(ctx, backupConfig{
DSN: dsn,
StorageDir: srcStorageDir,
Driver: srcDriver,
TenantSlug: drillTenantSlug,
OutDir: backupDir,
PgSchema: srcSchema,
@@ -128,11 +151,13 @@ func TestBackupRestoreDrill_RealRoundTrip(t *testing.T) {
dstSchema := fmt.Sprintf("fdn27dst_%d", time.Now().UnixNano())
t.Cleanup(func() { _, _ = admin.Exec(context.Background(), "DROP SCHEMA IF EXISTS "+dstSchema+" CASCADE") })
dstStorageDir := t.TempDir()
dstDriver := drillDriver(t, "TEST_S3_BUCKET", dstStorageDir)
restoreStart := time.Now()
if err := runRestore(ctx, restoreConfig{
DSN: dsn,
StorageDir: dstStorageDir,
Driver: dstDriver,
InDir: backupDir,
TargetSchema: dstSchema,
Crypto: crypt,
@@ -162,10 +187,9 @@ func TestBackupRestoreDrill_RealRoundTrip(t *testing.T) {
// -- FDN-23-Integritaetsjob auf dem wiederhergestellten Bestand: muss
// "ok" liefern, sonst waere der Restore nur scheinbar vollstaendig --
dstStorageSvc := storage.NewLocalDriver(dstStorageDir, []byte("secret"), "https://files.example.test", 0)
verifier := &integrity.Verifier{
Pool: dstPool,
Storage: storage.NewService(dstStorageSvc, drillNoopUsage{}, drillTenantSlug),
Storage: storage.NewService(dstDriver, drillNoopUsage{}, drillTenantSlug),
Crypto: crypt,
TenantSlug: drillTenantSlug,
Config: integrity.Config{Mode: integrity.ModeFull, Batch: 100},
dms/cmd/backup-tool/main.go
+9 -2
View File
@@ -7,6 +7,7 @@ import (
"os"
dmscrypto "gitea.perlbach24.de/scripte/nexarch/dms/internal/crypto"
"gitea.perlbach24.de/scripte/nexarch/dms/internal/storage"
)
func main() {
@@ -18,9 +19,15 @@ func main() {
var err error
switch os.Args[1] {
case "backup":
err = runBackup(ctx, backupConfigFromEnv(os.Args[2]))
cfg := backupConfigFromEnv(os.Args[2])
if cfg.Driver, err = storage.NewDriverFromEnv(ctx, "NEXARCH_DMS_BACKUP"); err == nil {
err = runBackup(ctx, cfg)
}
case "restore":
err = runRestore(ctx, restoreConfigFromEnv(os.Args[2]))
cfg := restoreConfigFromEnv(os.Args[2])
if cfg.Driver, err = storage.NewDriverFromEnv(ctx, "NEXARCH_DMS_RESTORE"); err == nil {
err = runRestore(ctx, cfg)
}
default:
usage()
os.Exit(2)
dms/cmd/backup-tool/restore.go
+5 -1
View File
@@ -23,6 +23,7 @@ var errTargetNotEmpty = errors.New("restore: zieldatenbank ist nicht leer (--for
type restoreConfig struct {
DSN string
StorageDir string
Driver storage.Driver
InDir string
PgRestorePath string
PsqlPath string
@@ -69,7 +70,10 @@ func runRestore(ctx context.Context, cfg restoreConfig) error {
}
}
driver := storage.NewLocalDriver(cfg.StorageDir, []byte("backup-tool-unused"), "", 0)
driver := cfg.Driver
if driver == nil {
driver = storage.NewLocalDriver(cfg.StorageDir, []byte("backup-tool-unused"), "", 0)
}
for _, obj := range m.Objects {
if err := restoreObject(ctx, driver, cfg.InDir, obj); err != nil {
return fmt.Errorf("restore: objekt %q (revision %s): %w", obj.StorageKey, obj.RevisionID, err)