sysops
1fedd683e0
Stand: agent-06 (Audit-Log), agent-05 (Krankmeldung), agent-07 Phase 1 (Personalnummer), Busylight-Pull-Integration, TOTP/2FA, Abwesenheiten, Zeiterfassung, Kiosk-Grundgerüst. Migrations 0001–0023 deployed auf 192.168.1.137 + .164. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
629 lines
24 KiB
Bash
629 lines
24 KiB
Bash
#!/bin/bash
|
||
# =============================================================================
|
||
# TimeMaster – Umfassendes Update- und Health-Check-Script
|
||
# Server: root@192.168.1.137
|
||
# =============================================================================
|
||
# Phasen:
|
||
# 0 Pre-Flight Check
|
||
# 1 Backup (DB-Dump, pip freeze, 30-Tage-Retention)
|
||
# 2 System-Update (apt) + Erkennung PG/Python-Versionssprung
|
||
# 3 Python-Abhängigkeiten prüfen und aktualisieren
|
||
# 4 Post-Update Funktionstest (API-Endpunkte + nginx)
|
||
# 5 Performance-Test (curl, psql, redis-cli)
|
||
# 6 Zusammenfassung
|
||
# =============================================================================
|
||
|
||
# --- Globale Einstellungen ---------------------------------------------------
|
||
APP_DIR="/opt/timemaster"
|
||
BACKEND_DIR="$APP_DIR/backend"
|
||
VENV="$BACKEND_DIR/venv"
|
||
REQ="$BACKEND_DIR/requirements.txt"
|
||
BACKUP_DIR="/tank/backup"
|
||
DB_NAME="timemaster_db"
|
||
API_BASE="http://localhost:8000"
|
||
FRONTEND_BASE="http://localhost"
|
||
MIN_FREE_MB=500
|
||
PERF_RUNS=5 # Anzahl Messungen für Durchschnitt
|
||
EXIT_CODE=0 # wird auf 1 gesetzt bei Problemen
|
||
|
||
# --- Farben ------------------------------------------------------------------
|
||
RED='\033[0;31m'
|
||
YELLOW='\033[1;33m'
|
||
GREEN='\033[0;32m'
|
||
CYAN='\033[0;36m'
|
||
BOLD='\033[1m'
|
||
RESET='\033[0m'
|
||
|
||
# --- Hilfsfunktionen ---------------------------------------------------------
|
||
ok() { echo -e "${GREEN}✅ $*${RESET}"; }
|
||
warn() { echo -e "${YELLOW}⚠️ $*${RESET}"; EXIT_CODE=1; }
|
||
err() { echo -e "${RED}❌ $*${RESET}"; EXIT_CODE=1; }
|
||
info() { echo -e "${CYAN}ℹ $*${RESET}"; }
|
||
hdr() { echo -e "\n${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}"; \
|
||
echo -e "${BOLD} $*${RESET}"; \
|
||
echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}"; }
|
||
|
||
# Misst Response-Zeit eines curl-Aufrufs in ms, gibt HTTP-Status und Zeit aus
|
||
# Aufruf: measure_curl <url>
|
||
# Ausgabe (stdout, TAB-getrennt): HTTP_STATUS<TAB>MS
|
||
measure_curl() {
|
||
local url="$1"
|
||
local result
|
||
result=$(curl -s -o /dev/null -w "%{http_code}\t%{time_total}" \
|
||
--connect-timeout 5 --max-time 10 "$url" 2>/dev/null || echo "000\t0")
|
||
local http_code ms_float
|
||
http_code=$(echo "$result" | cut -f1)
|
||
ms_float=$(echo "$result" | cut -f2)
|
||
# Sekunden → Millisekunden (ganzzahlig)
|
||
local ms
|
||
ms=$(awk "BEGIN {printf \"%d\", $ms_float * 1000}")
|
||
echo "${http_code}"$'\t'"${ms}"
|
||
}
|
||
|
||
# Durchschnitt aus N curl-Messungen (nur ms)
|
||
avg_curl_ms() {
|
||
local url="$1"
|
||
local n="${2:-$PERF_RUNS}"
|
||
local total=0
|
||
local i
|
||
for (( i=0; i<n; i++ )); do
|
||
local res
|
||
res=$(measure_curl "$url")
|
||
local ms
|
||
ms=$(echo "$res" | cut -f2)
|
||
total=$(( total + ms ))
|
||
done
|
||
echo $(( total / n ))
|
||
}
|
||
|
||
# Farbe für Performance-Ergebnis
|
||
perf_color() {
|
||
local ms="$1" threshold="$2"
|
||
if [[ "$ms" -lt "$threshold" ]]; then echo "${GREEN}";
|
||
elif [[ "$ms" -lt $(( threshold * 3 )) ]]; then echo "${YELLOW}";
|
||
else echo "${RED}"; fi
|
||
}
|
||
|
||
perf_label() {
|
||
local ms="$1" threshold="$2"
|
||
if [[ "$ms" -lt "$threshold" ]]; then echo "OK";
|
||
elif [[ "$ms" -lt $(( threshold * 3 )) ]]; then echo "LANGSAM";
|
||
else echo "FEHLER"; fi
|
||
}
|
||
|
||
# Sammle Zusammenfassungs-Zeilen
|
||
SUMMARY_LINES=()
|
||
summary_add() { SUMMARY_LINES+=("$1"); }
|
||
|
||
# =============================================================================
|
||
# PHASE 0 – PRE-FLIGHT CHECK
|
||
# =============================================================================
|
||
hdr "Phase 0 · Pre-Flight Check"
|
||
|
||
PREFLIGHT_OK=true
|
||
|
||
# --- Services prüfen ---------------------------------------------------------
|
||
for SVC in timemaster nginx postgresql redis; do
|
||
if systemctl is-active --quiet "$SVC" 2>/dev/null; then
|
||
ok "Service $SVC läuft"
|
||
else
|
||
err "Service $SVC ist NICHT aktiv"
|
||
PREFLIGHT_OK=false
|
||
fi
|
||
done
|
||
|
||
# --- Festplattenplatz --------------------------------------------------------
|
||
FREE_MB=$(df -m / | awk 'NR==2 {print $4}')
|
||
if [[ "$FREE_MB" -ge "$MIN_FREE_MB" ]]; then
|
||
ok "Festplatte: ${FREE_MB} MB frei (Minimum: ${MIN_FREE_MB} MB)"
|
||
else
|
||
err "Zu wenig Festplattenplatz: ${FREE_MB} MB frei (Minimum: ${MIN_FREE_MB} MB)"
|
||
PREFLIGHT_OK=false
|
||
fi
|
||
|
||
# --- PostgreSQL erreichbar ---------------------------------------------------
|
||
if su -c "psql -d $DB_NAME -c 'SELECT 1' -q --tuples-only" postgres &>/dev/null; then
|
||
ok "PostgreSQL erreichbar (DB: $DB_NAME)"
|
||
else
|
||
err "PostgreSQL NICHT erreichbar (DB: $DB_NAME)"
|
||
PREFLIGHT_OK=false
|
||
fi
|
||
|
||
# --- Redis erreichbar --------------------------------------------------------
|
||
if redis-cli ping 2>/dev/null | grep -q "PONG"; then
|
||
ok "Redis erreichbar"
|
||
else
|
||
err "Redis NICHT erreichbar"
|
||
PREFLIGHT_OK=false
|
||
fi
|
||
|
||
# --- API Health-Endpoint -----------------------------------------------------
|
||
API_PRE=$(measure_curl "$API_BASE/health")
|
||
API_PRE_STATUS=$(echo "$API_PRE" | cut -f1)
|
||
if [[ "$API_PRE_STATUS" == "200" ]]; then
|
||
ok "API /health antwortet mit HTTP 200"
|
||
else
|
||
err "API /health antwortet NICHT (HTTP $API_PRE_STATUS)"
|
||
PREFLIGHT_OK=false
|
||
fi
|
||
|
||
# --- Abbruch bei Pre-Flight-Fehler ------------------------------------------
|
||
if ! $PREFLIGHT_OK; then
|
||
echo ""
|
||
err "Pre-Flight Check FEHLGESCHLAGEN. Script wird abgebrochen."
|
||
err "Diagnose-Befehle:"
|
||
err " journalctl -u timemaster -n 50"
|
||
err " journalctl -u nginx -n 20"
|
||
err " systemctl status postgresql"
|
||
err " redis-cli ping"
|
||
exit 1
|
||
fi
|
||
|
||
summary_add "${GREEN}Phase 0: Pre-Flight Check OK${RESET}"
|
||
|
||
# =============================================================================
|
||
# PHASE 1 – BACKUP
|
||
# =============================================================================
|
||
hdr "Phase 1 · Backup"
|
||
|
||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||
mkdir -p "$BACKUP_DIR"
|
||
|
||
# PostgreSQL Dump
|
||
DB_DUMP="$BACKUP_DIR/db_${TIMESTAMP}.dump"
|
||
info "Erstelle PostgreSQL-Dump → $DB_DUMP"
|
||
if su -c "pg_dump -Fc $DB_NAME" postgres > "$DB_DUMP" 2>/tmp/pg_dump_err; then
|
||
DUMP_SIZE=$(du -sh "$DB_DUMP" | cut -f1)
|
||
ok "DB-Dump erstellt: $DB_DUMP ($DUMP_SIZE)"
|
||
summary_add "${GREEN}Phase 1: DB-Backup OK ($DUMP_SIZE)${RESET}"
|
||
else
|
||
warn "DB-Dump FEHLGESCHLAGEN: $(cat /tmp/pg_dump_err)"
|
||
summary_add "${YELLOW}Phase 1: DB-Backup FEHLGESCHLAGEN${RESET}"
|
||
fi
|
||
|
||
# pip freeze sichern
|
||
PIP_FREEZE="$BACKUP_DIR/pip_freeze_${TIMESTAMP}.txt"
|
||
info "Sichere pip freeze → $PIP_FREEZE"
|
||
if "$VENV/bin/pip" freeze > "$PIP_FREEZE" 2>/dev/null; then
|
||
ok "pip freeze gesichert: $PIP_FREEZE"
|
||
else
|
||
warn "pip freeze konnte nicht gesichert werden"
|
||
fi
|
||
|
||
# 30-Tage-Retention (DB-Dumps)
|
||
info "Lösche DB-Dumps älter als 30 Tage..."
|
||
find "$BACKUP_DIR" -name "db_*.dump" -mtime +30 -delete 2>/dev/null && ok "Alte Backups bereinigt" || true
|
||
# pip-freeze-Dateien ebenfalls bereinigen
|
||
find "$BACKUP_DIR" -name "pip_freeze_*.txt" -mtime +30 -delete 2>/dev/null || true
|
||
|
||
# =============================================================================
|
||
# PHASE 2 – SYSTEM-UPDATE (apt)
|
||
# =============================================================================
|
||
hdr "Phase 2 · System-Update (apt)"
|
||
|
||
# Versionen VOR Update sichern
|
||
PG_BEFORE=$(pg_lsclusters -h 2>/dev/null | awk 'NR==1 {print $1}' || echo "unbekannt")
|
||
PYTHON_BEFORE_FULL=$(grep "^version = " "$VENV/pyvenv.cfg" 2>/dev/null | cut -d' ' -f3 || echo "unbekannt")
|
||
PYTHON_BEFORE_MAJOR=$(echo "$PYTHON_BEFORE_FULL" | cut -d. -f1-2)
|
||
|
||
info "PostgreSQL vor Update : $PG_BEFORE"
|
||
info "Python (venv) vor Update: $PYTHON_BEFORE_FULL (Major: $PYTHON_BEFORE_MAJOR)"
|
||
|
||
# apt-Operationen (kein set -e hier, damit Fehler nicht alles abbricht)
|
||
info "Führe apt update aus..."
|
||
APT_UPDATE_OUT=$(apt update 2>&1)
|
||
APT_UPDATE_RC=$?
|
||
if [[ $APT_UPDATE_RC -eq 0 ]]; then
|
||
ok "apt update erfolgreich"
|
||
else
|
||
warn "apt update mit Warnungen: $(echo "$APT_UPDATE_OUT" | tail -3)"
|
||
fi
|
||
|
||
info "Führe apt upgrade -y aus (kann etwas dauern)..."
|
||
APT_UPGRADE_OUT=$(DEBIAN_FRONTEND=noninteractive apt upgrade -y 2>&1)
|
||
APT_UPGRADE_RC=$?
|
||
# Zähle aktualisierte Pakete
|
||
APT_UPGRADED=$(echo "$APT_UPGRADE_OUT" | grep -oP '^\d+ upgraded' | grep -oP '^\d+' || echo "0")
|
||
APT_INSTALLED=$(echo "$APT_UPGRADE_OUT" | grep -oP '\d+ newly installed' | grep -oP '^\d+' || echo "0")
|
||
|
||
if [[ $APT_UPGRADE_RC -eq 0 ]]; then
|
||
ok "apt upgrade erfolgreich (${APT_UPGRADED:-0} Pakete aktualisiert, ${APT_INSTALLED:-0} neu installiert)"
|
||
summary_add "${GREEN}Phase 2: apt upgrade OK (${APT_UPGRADED:-0} Pakete)${RESET}"
|
||
else
|
||
warn "apt upgrade mit Fehler (RC=$APT_UPGRADE_RC)"
|
||
summary_add "${YELLOW}Phase 2: apt upgrade mit Warnungen${RESET}"
|
||
fi
|
||
|
||
info "Führe apt autoremove -y aus..."
|
||
DEBIAN_FRONTEND=noninteractive apt autoremove -y &>/dev/null && ok "apt autoremove OK" || warn "apt autoremove mit Fehler"
|
||
|
||
# --- PostgreSQL Major-Versionssprung erkennen --------------------------------
|
||
PG_AFTER=$(pg_lsclusters -h 2>/dev/null | awk 'NR==1 {print $1}' || echo "unbekannt")
|
||
info "PostgreSQL nach Update: $PG_AFTER"
|
||
|
||
if [[ "$PG_BEFORE" != "$PG_AFTER" && "$PG_BEFORE" != "unbekannt" ]]; then
|
||
warn "PostgreSQL Major-Versionssprung erkannt: $PG_BEFORE → $PG_AFTER"
|
||
echo ""
|
||
echo -e "${YELLOW} Das Backup liegt in: $BACKUP_DIR${RESET}"
|
||
echo -e "${YELLOW} Verfügbare Cluster:${RESET}"
|
||
pg_lsclusters
|
||
echo ""
|
||
read -r -p " pg_upgradecluster $PG_BEFORE main → $PG_AFTER jetzt ausführen? [j/N] " PG_ANSWER
|
||
if [[ "$PG_ANSWER" =~ ^[jJ]$ ]]; then
|
||
info "Stoppe timemaster Service..."
|
||
systemctl stop timemaster
|
||
info "Führe pg_upgradecluster $PG_BEFORE main aus..."
|
||
if pg_upgradecluster "$PG_BEFORE" main; then
|
||
ok "pg_upgradecluster erfolgreich"
|
||
summary_add "${GREEN}Phase 2: PostgreSQL-Cluster-Upgrade $PG_BEFORE→$PG_AFTER OK${RESET}"
|
||
else
|
||
err "pg_upgradecluster FEHLGESCHLAGEN – manuell eingreifen!"
|
||
summary_add "${RED}Phase 2: PostgreSQL-Upgrade FEHLGESCHLAGEN${RESET}"
|
||
fi
|
||
info "Starte timemaster Service..."
|
||
systemctl start timemaster
|
||
sleep 3
|
||
else
|
||
warn "pg_upgradecluster übersprungen. Alter Cluster $PG_BEFORE ist noch aktiv!"
|
||
warn "Bitte manuell migrieren: pg_upgradecluster $PG_BEFORE main"
|
||
summary_add "${YELLOW}Phase 2: PostgreSQL-Upgrade übersprungen${RESET}"
|
||
fi
|
||
else
|
||
ok "PostgreSQL: kein Major-Versionssprung ($PG_AFTER)"
|
||
summary_add "${GREEN}Phase 2: PostgreSQL stabil (Version $PG_AFTER)${RESET}"
|
||
fi
|
||
|
||
# --- Python Major-Versionssprung erkennen ------------------------------------
|
||
PYTHON_AFTER_FULL=$(python3 --version 2>/dev/null | cut -d' ' -f2 || echo "unbekannt")
|
||
PYTHON_AFTER_MAJOR=$(echo "$PYTHON_AFTER_FULL" | cut -d. -f1-2)
|
||
|
||
info "Python nach Update: $PYTHON_AFTER_FULL (Major: $PYTHON_AFTER_MAJOR)"
|
||
|
||
if [[ "$PYTHON_BEFORE_MAJOR" != "$PYTHON_AFTER_MAJOR" && "$PYTHON_BEFORE_MAJOR" != "unbekannt" ]]; then
|
||
warn "Python Major-Versionssprung: $PYTHON_BEFORE_MAJOR → $PYTHON_AFTER_MAJOR"
|
||
info "Baue venv automatisch neu..."
|
||
|
||
systemctl stop timemaster || true
|
||
|
||
cd "$BACKEND_DIR"
|
||
mv venv "venv.old.${PYTHON_BEFORE_MAJOR}" 2>/dev/null || true
|
||
|
||
if python3 -m venv venv; then
|
||
"$VENV/bin/pip" install --upgrade pip -q
|
||
if "$VENV/bin/pip" install -r "$REQ" -q; then
|
||
ok "venv neu gebaut mit Python $PYTHON_AFTER_FULL"
|
||
summary_add "${GREEN}Phase 2: venv neu gebaut (Python $PYTHON_BEFORE_MAJOR→$PYTHON_AFTER_MAJOR)${RESET}"
|
||
else
|
||
err "pip install -r requirements.txt FEHLGESCHLAGEN"
|
||
summary_add "${RED}Phase 2: venv-Neubau FEHLGESCHLAGEN${RESET}"
|
||
fi
|
||
else
|
||
err "python3 -m venv fehlgeschlagen"
|
||
summary_add "${RED}Phase 2: venv-Neubau FEHLGESCHLAGEN${RESET}"
|
||
fi
|
||
|
||
systemctl start timemaster || true
|
||
sleep 3
|
||
else
|
||
ok "Python: kein Major-Versionssprung ($PYTHON_AFTER_MAJOR)"
|
||
summary_add "${GREEN}Phase 2: Python stabil (Version $PYTHON_AFTER_FULL)${RESET}"
|
||
fi
|
||
|
||
# =============================================================================
|
||
# PHASE 3 – PYTHON-ABHÄNGIGKEITEN PRÜFEN UND AKTUALISIEREN
|
||
# =============================================================================
|
||
hdr "Phase 3 · Python-Abhängigkeiten"
|
||
|
||
if [[ ! -f "$REQ" ]]; then
|
||
warn "requirements.txt nicht gefunden: $REQ"
|
||
summary_add "${YELLOW}Phase 3: requirements.txt fehlt${RESET}"
|
||
else
|
||
# Installierte Pakete als assoziatives Array: name → version (lowercase)
|
||
# pip list --format=freeze liefert "Paketname==Version" – cut auf erstes == aufteilen
|
||
declare -A INSTALLED_PKGS
|
||
while IFS= read -r line; do
|
||
[[ -z "$line" || "$line" =~ ^# ]] && continue
|
||
pkg=$(echo "$line" | cut -d= -f1)
|
||
ver=$(echo "$line" | cut -d= -f3) # f1=name, f2="" (wegen ==), f3=version
|
||
pname=$(echo "$pkg" | tr '[:upper:]' '[:lower:]' | tr '_' '-' | xargs)
|
||
pver=$(echo "$ver" | xargs)
|
||
[[ -n "$pname" && -n "$pver" ]] && INSTALLED_PKGS["$pname"]="$pver"
|
||
done < <("$VENV/bin/pip" list --format=freeze 2>/dev/null)
|
||
|
||
MISSING_PKGS=()
|
||
WRONG_VERSION_PKGS=()
|
||
OK_PKGS=0
|
||
|
||
# requirements.txt parsen (ignoriere Kommentare, leere Zeilen, git+-Zeilen)
|
||
while IFS= read -r line; do
|
||
# Leere Zeilen und Kommentare überspringen
|
||
[[ -z "$line" || "$line" =~ ^# ]] && continue
|
||
# git+https etc. überspringen
|
||
[[ "$line" =~ ^-e|^git\+ ]] && continue
|
||
|
||
# Paketname und Version extrahieren
|
||
req_name=$(echo "$line" | grep -oP '^[A-Za-z0-9_.-]+' | tr '[:upper:]' '[:lower:]' | tr '_' '-')
|
||
req_op=$(echo "$line" | grep -oP '(==|>=|<=|~=|!=)' | head -1 || true)
|
||
req_ver=$(echo "$line" | grep -oP '(==|>=|<=|~=|!=)\K[0-9A-Za-z._-]+' | head -1 || true)
|
||
|
||
[[ -z "$req_name" ]] && continue
|
||
|
||
inst_ver="${INSTALLED_PKGS[$req_name]:-}"
|
||
|
||
if [[ -z "$inst_ver" ]]; then
|
||
MISSING_PKGS+=("$line")
|
||
elif [[ -n "$req_op" && "$req_op" == "==" && "$inst_ver" != "$req_ver" ]]; then
|
||
WRONG_VERSION_PKGS+=("$req_name==$req_ver (installiert: $inst_ver)")
|
||
else
|
||
(( OK_PKGS++ )) || true
|
||
fi
|
||
done < "$REQ"
|
||
|
||
ok "$OK_PKGS Pakete bereits korrekt installiert"
|
||
|
||
CHANGED_PKGS=()
|
||
|
||
if [[ ${#MISSING_PKGS[@]} -gt 0 ]]; then
|
||
echo -e "${YELLOW}⚠️ ${#MISSING_PKGS[@]} fehlende Pakete gefunden – werden jetzt installiert:${RESET}"
|
||
for p in "${MISSING_PKGS[@]}"; do info " + $p"; done
|
||
|
||
info "Stoppe timemaster Service für pip install..."
|
||
systemctl stop timemaster || true
|
||
|
||
INSTALL_FAILED=()
|
||
for p in "${MISSING_PKGS[@]}"; do
|
||
info " Installiere: $p"
|
||
if "$VENV/bin/pip" install "$p" -q; then
|
||
ok " Installiert: $p"
|
||
CHANGED_PKGS+=("+ $p")
|
||
else
|
||
err " FEHLER bei Installation: $p"
|
||
INSTALL_FAILED+=("$p")
|
||
fi
|
||
done
|
||
|
||
info "Starte timemaster Service..."
|
||
systemctl start timemaster || true
|
||
sleep 3
|
||
|
||
# Nur Exit-Code setzen wenn Installation tatsächlich fehlschlug
|
||
if [[ ${#INSTALL_FAILED[@]} -gt 0 ]]; then
|
||
warn "${#INSTALL_FAILED[@]} Pakete konnten NICHT installiert werden"
|
||
else
|
||
ok "Alle fehlenden Pakete erfolgreich nachinstalliert"
|
||
fi
|
||
else
|
||
ok "Keine fehlenden Pakete"
|
||
fi
|
||
|
||
if [[ ${#WRONG_VERSION_PKGS[@]} -gt 0 ]]; then
|
||
echo -e "${YELLOW}⚠️ ${#WRONG_VERSION_PKGS[@]} Pakete mit falscher Version – werden aktualisiert:${RESET}"
|
||
for p in "${WRONG_VERSION_PKGS[@]}"; do info " ~ $p"; done
|
||
|
||
# Service nur stoppen wenn nicht schon gestoppt
|
||
if systemctl is-active --quiet timemaster 2>/dev/null; then
|
||
info "Stoppe timemaster Service für pip upgrade..."
|
||
systemctl stop timemaster || true
|
||
RESTART_AFTER_PIP=true
|
||
else
|
||
RESTART_AFTER_PIP=false
|
||
fi
|
||
|
||
UPGRADE_FAILED=()
|
||
for entry in "${WRONG_VERSION_PKGS[@]}"; do
|
||
pkg_spec=$(echo "$entry" | cut -d' ' -f1)
|
||
info " Aktualisiere: $pkg_spec"
|
||
if "$VENV/bin/pip" install "$pkg_spec" -q; then
|
||
ok " Aktualisiert: $pkg_spec"
|
||
CHANGED_PKGS+=("~ $pkg_spec")
|
||
else
|
||
err " FEHLER beim Update: $pkg_spec"
|
||
UPGRADE_FAILED+=("$pkg_spec")
|
||
fi
|
||
done
|
||
|
||
if $RESTART_AFTER_PIP; then
|
||
info "Starte timemaster Service..."
|
||
systemctl start timemaster || true
|
||
sleep 3
|
||
fi
|
||
|
||
if [[ ${#UPGRADE_FAILED[@]} -gt 0 ]]; then
|
||
warn "${#UPGRADE_FAILED[@]} Pakete konnten NICHT aktualisiert werden"
|
||
else
|
||
ok "Alle Versionskonflikte erfolgreich behoben"
|
||
fi
|
||
else
|
||
ok "Alle Paketversionen korrekt"
|
||
fi
|
||
|
||
if [[ ${#CHANGED_PKGS[@]} -gt 0 ]]; then
|
||
summary_add "${GREEN}Phase 3: ${#CHANGED_PKGS[@]} Pakete geändert:${RESET}"
|
||
for c in "${CHANGED_PKGS[@]}"; do
|
||
summary_add " ${CYAN}$c${RESET}"
|
||
done
|
||
else
|
||
summary_add "${GREEN}Phase 3: Keine Paketänderungen notwendig${RESET}"
|
||
fi
|
||
fi
|
||
|
||
# =============================================================================
|
||
# PHASE 4 – POST-UPDATE FUNKTIONSTEST
|
||
# =============================================================================
|
||
hdr "Phase 4 · Post-Update Funktionstest"
|
||
|
||
# Service sicherheitshalber aktivieren
|
||
if ! systemctl is-active --quiet timemaster 2>/dev/null; then
|
||
info "Starte timemaster Service..."
|
||
systemctl start timemaster || true
|
||
sleep 5
|
||
fi
|
||
|
||
FUNC_OK=true
|
||
|
||
# GET /health → 200
|
||
RES=$(measure_curl "$API_BASE/health")
|
||
STATUS=$(echo "$RES" | cut -f1)
|
||
MS=$(echo "$RES" | cut -f2)
|
||
if [[ "$STATUS" == "200" ]]; then
|
||
ok "GET /health → HTTP $STATUS (${MS}ms)"
|
||
else
|
||
err "GET /health → HTTP $STATUS (erwartet: 200)"
|
||
FUNC_OK=false
|
||
fi
|
||
|
||
# GET /api/v1/auth/me → 401 (kein Token)
|
||
RES=$(measure_curl "$API_BASE/api/v1/auth/me")
|
||
STATUS=$(echo "$RES" | cut -f1)
|
||
MS=$(echo "$RES" | cut -f2)
|
||
if [[ "$STATUS" == "401" ]]; then
|
||
ok "GET /api/v1/auth/me → HTTP $STATUS (kein Token = korrekt, ${MS}ms)"
|
||
else
|
||
err "GET /api/v1/auth/me → HTTP $STATUS (erwartet: 401)"
|
||
FUNC_OK=false
|
||
fi
|
||
|
||
# GET / nginx Frontend → 200 mit HTML
|
||
NGINX_RES=$(curl -s -o /tmp/nginx_root.html -w "%{http_code}" \
|
||
--connect-timeout 5 --max-time 10 "$FRONTEND_BASE/" 2>/dev/null || echo "000")
|
||
if [[ "$NGINX_RES" == "200" ]]; then
|
||
# Prüfe ob HTML zurückkommt
|
||
if grep -qi "<html" /tmp/nginx_root.html 2>/dev/null; then
|
||
ok "GET / → HTTP 200 mit HTML (nginx liefert Frontend aus)"
|
||
else
|
||
warn "GET / → HTTP 200, aber kein HTML-Inhalt (prüfe nginx-Konfiguration)"
|
||
fi
|
||
else
|
||
err "GET / → HTTP $NGINX_RES (nginx Frontend nicht erreichbar)"
|
||
FUNC_OK=false
|
||
fi
|
||
rm -f /tmp/nginx_root.html
|
||
|
||
if $FUNC_OK; then
|
||
summary_add "${GREEN}Phase 4: Alle Funktionstests bestanden${RESET}"
|
||
else
|
||
summary_add "${RED}Phase 4: Funktionstests FEHLGESCHLAGEN${RESET}"
|
||
fi
|
||
|
||
# =============================================================================
|
||
# PHASE 5 – PERFORMANCE-TEST
|
||
# =============================================================================
|
||
hdr "Phase 5 · Performance-Test (je $PERF_RUNS Messungen)"
|
||
|
||
# Tabellen-Header
|
||
printf "\n${BOLD}%-40s %14s %10s${RESET}\n" "Endpunkt / Ressource" "Ø Zeit (ms)" "Status"
|
||
printf "%-40s %14s %10s\n" "$(printf '%0.s─' {1..40})" "$(printf '%0.s─' {1..14})" "$(printf '%0.s─' {1..10})"
|
||
|
||
PERF_ISSUES=0
|
||
|
||
# Funktion: Zeile in Tabelle ausgeben
|
||
print_perf_row() {
|
||
local label="$1" ms="$2" threshold="$3"
|
||
local label_status
|
||
label_status=$(perf_label "$ms" "$threshold")
|
||
local col
|
||
col=$(perf_color "$ms" "$threshold")
|
||
if [[ "$label_status" != "OK" ]]; then (( PERF_ISSUES++ )) || true; fi
|
||
printf "${col}%-40s %14s %10s${RESET}\n" "$label" "${ms}ms" "$label_status"
|
||
}
|
||
|
||
# GET /health (Schwellwert 100ms)
|
||
info "Messe GET /health ($PERF_RUNS Runs)..."
|
||
HEALTH_MS=$(avg_curl_ms "$API_BASE/health")
|
||
print_perf_row "GET /health" "$HEALTH_MS" 100
|
||
|
||
# GET /api/v1/auth/me (Schwellwert 200ms)
|
||
info "Messe GET /api/v1/auth/me ($PERF_RUNS Runs)..."
|
||
AUTH_MS=$(avg_curl_ms "$API_BASE/api/v1/auth/me")
|
||
print_perf_row "GET /api/v1/auth/me" "$AUTH_MS" 200
|
||
|
||
# PostgreSQL SELECT 1 (Schwellwert 100ms)
|
||
info "Messe PostgreSQL SELECT 1 ($PERF_RUNS Runs)..."
|
||
PG_TOTAL=0
|
||
for (( i=0; i<PERF_RUNS; i++ )); do
|
||
PG_START=$( date +%s%N )
|
||
su -c "psql -d $DB_NAME -c 'SELECT 1' -q --tuples-only" postgres &>/dev/null || true
|
||
PG_END=$( date +%s%N )
|
||
PG_DIFF=$(( (PG_END - PG_START) / 1000000 ))
|
||
PG_TOTAL=$(( PG_TOTAL + PG_DIFF ))
|
||
done
|
||
PG_AVG=$(( PG_TOTAL / PERF_RUNS ))
|
||
print_perf_row "PostgreSQL SELECT 1" "$PG_AVG" 100
|
||
|
||
# Redis PING (Schwellwert 50ms)
|
||
info "Messe Redis PING ($PERF_RUNS Runs)..."
|
||
REDIS_TOTAL=0
|
||
for (( i=0; i<PERF_RUNS; i++ )); do
|
||
R_START=$( date +%s%N )
|
||
redis-cli ping &>/dev/null || true
|
||
R_END=$( date +%s%N )
|
||
R_DIFF=$(( (R_END - R_START) / 1000000 ))
|
||
REDIS_TOTAL=$(( REDIS_TOTAL + R_DIFF ))
|
||
done
|
||
REDIS_AVG=$(( REDIS_TOTAL / PERF_RUNS ))
|
||
|
||
# Redis PONG prüfen
|
||
REDIS_PING_RESULT=$(redis-cli ping 2>/dev/null || echo "FEHLER")
|
||
if [[ "$REDIS_PING_RESULT" != "PONG" ]]; then
|
||
printf "${RED}%-40s %14s %10s${RESET}\n" "Redis PING" "${REDIS_AVG}ms" "FEHLER"
|
||
(( PERF_ISSUES++ )) || true
|
||
else
|
||
print_perf_row "Redis PING" "$REDIS_AVG" 50
|
||
fi
|
||
|
||
echo ""
|
||
if [[ $PERF_ISSUES -eq 0 ]]; then
|
||
ok "Performance: Alle Werte im grünen Bereich"
|
||
summary_add "${GREEN}Phase 5: Performance OK (health ${HEALTH_MS}ms, auth/me ${AUTH_MS}ms, PG ${PG_AVG}ms, Redis ${REDIS_AVG}ms)${RESET}"
|
||
else
|
||
warn "Performance: $PERF_ISSUES Wert(e) außerhalb des Schwellwerts"
|
||
summary_add "${YELLOW}Phase 5: Performance-Warnung ($PERF_ISSUES langsame Werte)${RESET}"
|
||
fi
|
||
|
||
# =============================================================================
|
||
# PHASE 6 – ZUSAMMENFASSUNG
|
||
# =============================================================================
|
||
hdr "Phase 6 · Zusammenfassung"
|
||
|
||
echo ""
|
||
for line in "${SUMMARY_LINES[@]}"; do
|
||
echo -e " $line"
|
||
done
|
||
|
||
echo ""
|
||
echo -e "${BOLD}Service-Status:${RESET}"
|
||
for SVC in timemaster nginx postgresql redis; do
|
||
if systemctl is-active --quiet "$SVC" 2>/dev/null; then
|
||
ok " $SVC"
|
||
else
|
||
err " $SVC (NICHT aktiv)"
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
echo -e "${BOLD}Log-Befehle (bei Problemen):${RESET}"
|
||
echo -e " ${CYAN}journalctl -u timemaster -n 100 --no-pager${RESET}"
|
||
echo -e " ${CYAN}journalctl -u nginx -n 50 --no-pager${RESET}"
|
||
echo -e " ${CYAN}journalctl -u postgresql -n 50 --no-pager${RESET}"
|
||
echo -e " ${CYAN}journalctl -u redis -n 20 --no-pager${RESET}"
|
||
echo -e " ${CYAN}pg_lsclusters${RESET}"
|
||
|
||
echo ""
|
||
echo -e "${BOLD}Backups:${RESET}"
|
||
echo -e " Verzeichnis: $BACKUP_DIR"
|
||
ls -lh "$BACKUP_DIR"/*.dump 2>/dev/null | tail -5 || echo " (keine Dumps gefunden)"
|
||
|
||
echo ""
|
||
FINISH_TIME=$(date)
|
||
echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}"
|
||
if [[ $EXIT_CODE -eq 0 ]]; then
|
||
echo -e "${GREEN}${BOLD}✅ Update & Health-Check ERFOLGREICH abgeschlossen.${RESET}"
|
||
else
|
||
echo -e "${YELLOW}${BOLD}⚠️ Update & Health-Check abgeschlossen – PROBLEME GEFUNDEN.${RESET}"
|
||
echo -e "${YELLOW} Bitte obige Fehlermeldungen und Log-Befehle prüfen.${RESET}"
|
||
fi
|
||
echo -e " Fertig: $FINISH_TIME"
|
||
echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}"
|
||
|
||
exit $EXIT_CODE
|