account-api: DB-Start mit Backoff, Fehlerklassifizierung, Restart-Limits #410

Merged
patrick merged 1 commits from fix/account-api-start into main 2026-10-28 14:53:00 +01:00
4 changed files with 130 additions and 0 deletions

No files matched your search

+49
View File
@@ -0,0 +1,49 @@
package main
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5/pgconn"
)
const exitPermanentDBError = 2
type pinger interface {
Ping(ctx context.Context) error
}
func isPermanentDBError(err error) bool {
var pgErr *pgconn.PgError
if !errors.As(err, &pgErr) {
return false
}
switch pgErr.Code {
case "28000", "28P01", "3D000":
return true
}
return false
}
func waitForDB(ctx context.Context, p pinger, timeout time.Duration, sleep func(time.Duration)) error {
deadline := time.Now().Add(timeout)
delay := time.Second
for {
err := p.Ping(ctx)
if err == nil {
return nil
}
if isPermanentDBError(err) {
return fmt.Errorf("datenbank lehnt Anmeldung oder Datenbank ab (dauerhaft): %w", err)
}
if time.Now().Add(delay).After(deadline) {
return fmt.Errorf("datenbank nach %s nicht erreichbar: %w", timeout, err)
}
sleep(delay)
if delay < 30*time.Second {
delay *= 2
}
}
}
+66
View File
@@ -0,0 +1,66 @@
package main
import (
"context"
"errors"
"testing"
"time"
"github.com/jackc/pgx/v5/pgconn"
)
type fakePinger struct {
errs []error
calls int
}
func (f *fakePinger) Ping(context.Context) error {
i := f.calls
f.calls++
if i < len(f.errs) {
return f.errs[i]
}
return nil
}
func TestWaitForDB_PermanentAuthErrorStopsImmediately(t *testing.T) {
p := &fakePinger{errs: []error{&pgconn.PgError{Code: "28P01"}, nil}}
sleeps := 0
err := waitForDB(context.Background(), p, time.Minute, func(time.Duration) { sleeps++ })
if err == nil || !isPermanentDBError(err) {
t.Fatalf("erwartet dauerhaften fehler, war %v", err)
}
if p.calls != 1 || sleeps != 0 {
t.Fatalf("kein retry erwartet, calls=%d sleeps=%d", p.calls, sleeps)
}
}
func TestWaitForDB_TransientErrorRetriesWithBackoff(t *testing.T) {
p := &fakePinger{errs: []error{errors.New("connection refused"), errors.New("connection refused")}}
var delays []time.Duration
if err := waitForDB(context.Background(), p, time.Minute, func(d time.Duration) { delays = append(delays, d) }); err != nil {
t.Fatal(err)
}
if p.calls != 3 || len(delays) != 2 || delays[1] <= delays[0] {
t.Fatalf("calls=%d delays=%v", p.calls, delays)
}
}
func TestWaitForDB_GivesUpAfterTimeout(t *testing.T) {
p := &fakePinger{errs: []error{errors.New("x"), errors.New("x"), errors.New("x"), errors.New("x")}}
err := waitForDB(context.Background(), p, 0, func(time.Duration) {})
if err == nil || isPermanentDBError(err) {
t.Fatalf("erwartet transienten timeout-fehler, war %v", err)
}
}
func TestIsPermanentDBError(t *testing.T) {
for code, want := range map[string]bool{"28P01": true, "28000": true, "3D000": true, "57P03": false} {
if got := isPermanentDBError(&pgconn.PgError{Code: code}); got != want {
t.Errorf("%s: got %v", code, got)
}
}
if isPermanentDBError(errors.New("connection refused")) {
t.Error("verbindungsfehler ist transient")
}
}
+11
View File
@@ -82,6 +82,17 @@ func main() {
}
defer pool.Close()
dbWait := time.Duration(envInt("NEXARCH_ACCOUNT_DB_STARTUP_TIMEOUT_SECONDS", 120, 0, 3600)) * time.Second
if err := waitForDB(ctx, pool, dbWait, time.Sleep); err != nil {
log.Printf("account-api: %v — DSN in NEXARCH_ACCOUNT_TENANT_DSN (Rolle, Passwort, Datenbank) pruefen", err)
if isPermanentDBError(err) {
pool.Close()
os.Exit(exitPermanentDBError)
}
pool.Close()
os.Exit(1)
}
totpCipher, err := newTOTPCipher()
if err != nil {
log.Fatalf("totp-verschluesselung: %v", err)
@@ -1,6 +1,8 @@
[Unit]
Description=NEXARCH Core - Login/Account-Dienst (IAM-08/IAM-16)
After=network.target postgresql.service
StartLimitIntervalSec=300
StartLimitBurst=10
[Service]
Type=simple
@@ -8,6 +10,8 @@ User=nexarch
EnvironmentFile=/etc/nexarch/account-api.env
ExecStart=__INSTALL_DIR__/bin/account-api
Restart=on-failure
RestartSec=5
RestartPreventExitStatus=2
StandardOutput=journal
# QA-15 F-13/OPS-08: reiner Netzwerkdienst (HTTP + Postgres-DSN aus
# EnvironmentFile), keine eigene Schreibarbeit im Dateisystem.