454 lines
13 KiB
Bash
454 lines
13 KiB
Bash
#!/bin/bash
|
||
set -uo pipefail
|
||
IFS=$'\n\t'
|
||
|
||
PATH="/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
|
||
SCRIPT_NAME=$(basename "$0")
|
||
LOGFILE="/var/log/${SCRIPT_NAME%.sh}.log"
|
||
|
||
log() {
|
||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOGFILE"
|
||
}
|
||
|
||
usage() {
|
||
echo "Usage:"
|
||
echo " $0 -c /path/to/config # Full backup run"
|
||
echo " $0 [function] # Run individual function"
|
||
echo " $0 help # Show available functions"
|
||
exit 1
|
||
}
|
||
|
||
CONFIG_FILE=""
|
||
while getopts "c:" opt; do
|
||
case "$opt" in
|
||
c) CONFIG_FILE="$OPTARG" ;;
|
||
*) usage ;;
|
||
esac
|
||
done
|
||
shift $((OPTIND - 1))
|
||
|
||
if [[ -n "${CONFIG_FILE:-}" ]]; then
|
||
if [[ ! -f "$CONFIG_FILE" ]]; then
|
||
log "ERROR: Configuration file not found: $CONFIG_FILE"
|
||
exit 1
|
||
fi
|
||
|
||
if ! bash -n "$CONFIG_FILE"; then
|
||
log "Syntax error in configuration file $CONFIG_FILE"
|
||
exit 1
|
||
fi
|
||
|
||
source "$CONFIG_FILE"
|
||
|
||
REQUIRED_VARS=(
|
||
SOURCEPORT BACKUPSERVER ZSYNC MAINTDAY SHUTDOWN UPDATES
|
||
SOURCEHOST ZFSROOT ZFSSECOND ZFSTRGT ZPUSHTAG ZPUSHMINKEEP ZPUSHKEEP ZPUSHLABEL
|
||
PBSHOST BACKUPSTORE BACKUPSTOREPBS BACKUPEXCLUDE REPLEXCLUDE
|
||
)
|
||
|
||
MISSING_VARS=()
|
||
for var in "${REQUIRED_VARS[@]}"; do
|
||
if [[ -z "${!var:-}" ]]; then
|
||
MISSING_VARS+=("$var")
|
||
fi
|
||
done
|
||
|
||
if [[ "${#MISSING_VARS[@]}" -gt 0 ]]; then
|
||
log "Missing configuration variables:"
|
||
for var in "${MISSING_VARS[@]}"; do
|
||
log " - $var"
|
||
done
|
||
log "Aborting — please check your configuration file."
|
||
exit 1
|
||
else
|
||
log "All required configuration variables are set."
|
||
fi
|
||
fi
|
||
|
||
rssh() { ssh -p "$SOURCEPORT" "$@"; }
|
||
rscp() { scp -P "$SOURCEPORT" "$@"; }
|
||
|
||
get_sourcehostname() {
|
||
if [[ -z "${SOURCEHOSTNAME:-}" ]]; then
|
||
log "SOURCEHOSTNAME is empty, retrieving via SSH from $SOURCEHOST..."
|
||
SOURCEHOSTNAME=$(rssh "$SOURCEHOST" hostname)
|
||
log "Detected SOURCEHOSTNAME: $SOURCEHOSTNAME"
|
||
fi
|
||
}
|
||
|
||
set_wol_g_enabled() {
|
||
log "Checking if ethtool is installed..."
|
||
|
||
if ! command -v ethtool >/dev/null 2>&1; then
|
||
log "ethtool is not installed, attempting installation..."
|
||
apt update && apt install -y ethtool || {
|
||
log "Error: ethtool could not be installed."
|
||
return 1
|
||
}
|
||
else
|
||
log "ethtool is already installed."
|
||
fi
|
||
|
||
log "Setting Wake-on-LAN (WOL) to 'g' on interfaces with static IP..."
|
||
|
||
for iface in $(ls /sys/class/net | grep -vE '^(lo|tap|vmbr|veth|br|docker|bond|wl)'); do
|
||
if [[ -e "/sys/class/net/$iface/device" ]]; then
|
||
log "Processing physical interface: $iface"
|
||
|
||
current_wol=$(ethtool "$iface" 2>/dev/null | awk '/Wake-on:/ {print $2}')
|
||
if [[ "$current_wol" != "g" ]]; then
|
||
log "Setting WOL to 'g' for $iface..."
|
||
ethtool -s "$iface" wol g || log "Error setting WOL on $iface"
|
||
else
|
||
log "WOL already set to 'g' for $iface"
|
||
fi
|
||
|
||
if grep -qE "^\s*iface\s+$iface\s+inet\s+static" /etc/network/interfaces; then
|
||
if ! grep -A 5 -E "^\s*iface\s+$iface\s+inet\s+static" /etc/network/interfaces | grep -q "post-up ethtool -s $iface wol g"; then
|
||
log "Adding WOL command in static block for $iface..."
|
||
sed -i "/^\s*iface\s\+$iface\s\+inet\s\+static/a \ post-up ethtool -s $iface wol g" /etc/network/interfaces
|
||
else
|
||
log "WOL command already present in static block for $iface."
|
||
fi
|
||
else
|
||
log "No static entry found for $iface, no changes made."
|
||
fi
|
||
fi
|
||
done
|
||
}
|
||
|
||
write_zsync_config() {
|
||
get_sourcehostname
|
||
local conf_file="/etc/bashclub/$SOURCEHOST.conf"
|
||
log "Writing zsync config to $conf_file"
|
||
{
|
||
echo "target=$ZFSTRGT"
|
||
echo "source=root@$SOURCEHOST"
|
||
echo "sshport=$SOURCEPORT"
|
||
echo "tag=$ZPUSHTAG"
|
||
echo "snapshot_filter=\"$ZPUSHFILTER\""
|
||
echo "min_keep=$ZPUSHMINKEEP"
|
||
echo "zfs_auto_snapshot_keep=$ZPUSHKEEP"
|
||
echo "zfs_auto_snapshot_label=$ZPUSHLABEL"
|
||
echo "zfs_auto_snapshot_engine=internal"
|
||
echo "checkzfs_disabled=0"
|
||
echo "checkzfs_local=0"
|
||
echo "checkzfs_prefix=miyagi-$SOURCEHOSTNAME-$(hostname)-$ZPUSHTAG"
|
||
echo "checkzfs_max_age=1500,2000"
|
||
echo "checkzfs_max_snapshot_count=180,200"
|
||
echo "checkzfs_spool=1"
|
||
echo "checkzfs_spool_maxage=90000"
|
||
} > "$conf_file"
|
||
}
|
||
|
||
run_zsync() {
|
||
if [[ "$ZSYNC" != "no" ]]; then
|
||
/usr/bin/bashclub-zsync -c "/etc/bashclub/$SOURCEHOST.conf"
|
||
else
|
||
log "Zsync is disabled."
|
||
fi
|
||
}
|
||
|
||
run_remote_updates() {
|
||
if [[ "${UPDATES,,}" == "yes" ]]; then
|
||
log "Running updates on local system..."
|
||
apt update && apt dist-upgrade -y || log "Error during local updates"
|
||
|
||
if [[ "${BACKUPSERVER,,}" == "yes" ]]; then
|
||
log "Running updates on PBS host ($PBSHOST)..."
|
||
ssh root@"$PBSHOST" apt update && ssh root@"$PBSHOST" apt dist-upgrade -y || {
|
||
log "Error during updates on $PBSHOST"
|
||
}
|
||
else
|
||
log "PBS updates skipped (BACKUPSERVER=$BACKUPSERVER)"
|
||
fi
|
||
else
|
||
log "Updates disabled (UPDATES=$UPDATES)"
|
||
fi
|
||
}
|
||
|
||
send_piggyback() {
|
||
get_sourcehostname
|
||
local combined_host="miyagi-${SOURCEHOSTNAME}-$(hostname)"
|
||
local filename="90000_${combined_host}"
|
||
|
||
log "Reminder: Add a host named ${combined_host} in CMK (without Agent, Piggyback enabled)!"
|
||
log "Creating piggyback file: $filename"
|
||
|
||
{
|
||
echo "<<<<${combined_host}>>>>"
|
||
/usr/bin/check_mk_agent
|
||
echo "<<<<>>>>"
|
||
} > "$filename"
|
||
|
||
if rscp "$filename" "$SOURCEHOST:/var/lib/check_mk_agent/spool/"; then
|
||
log "Piggyback data successfully sent to $SOURCEHOST"
|
||
else
|
||
log "ERROR: Failed to send piggyback data to $SOURCEHOST"
|
||
fi
|
||
|
||
rm -f "$filename"
|
||
}
|
||
|
||
run_pbs_backup() {
|
||
if [[ "${BACKUPSERVER,,}" != "yes" ]]; then
|
||
log "PBS Backup übersprungen: BACKUPSERVER='$BACKUPSERVER' (muss 'yes' sein)"
|
||
return 0
|
||
fi
|
||
|
||
get_sourcehostname
|
||
log "Starte PBS Backup auf Host: $SOURCEHOST"
|
||
|
||
# PBS-Storage prüfen und ggf. aktivieren
|
||
log "Prüfe, ob PBS-Storage '$BACKUPSTORE' auf $SOURCEHOST aktiviert ist..."
|
||
if rssh root@"$SOURCEHOST" "pvesm status | grep -w '$BACKUPSTORE' | grep -q 'disabled'"; then
|
||
log "PBS-Storage '$BACKUPSTORE' ist deaktiviert – aktiviere temporär und warte 10 Sekunden..."
|
||
rssh root@"$SOURCEHOST" "pvesm set '$BACKUPSTORE' --disable 0 && sleep 10"
|
||
pbs_enabled_by_script=true
|
||
else
|
||
log "PBS-Storage '$BACKUPSTORE' ist bereits aktiviert."
|
||
pbs_enabled_by_script=false
|
||
fi
|
||
|
||
vzdump_success=false
|
||
|
||
# Versuch: vzdump mit Change Detection
|
||
## Eintrag muss in /etc/vzdump.conf entries-max 100000000
|
||
log "Starte vzdump mit Change Detection..."
|
||
if rssh root@"$SOURCEHOST" vzdump --pbs-change-detection-mode metadata \
|
||
--node "$SOURCEHOSTNAME" --storage "$BACKUPSTORE" \
|
||
--exclude "$BACKUPEXCLUDE" --mode snapshot --all 1 \
|
||
--notes-template '{{guestname}}'; then
|
||
|
||
log "vzdump (Change Detection) erfolgreich."
|
||
vzdump_success=true
|
||
else
|
||
log "vzdump (Change Detection) fehlgeschlagen – versuche Fallback ohne Change Detection..."
|
||
|
||
if rssh root@"$SOURCEHOST" vzdump \
|
||
--node "$SOURCEHOSTNAME" --storage "$BACKUPSTORE" \
|
||
--exclude "$BACKUPEXCLUDE" --mode snapshot --all 1 \
|
||
--notes-template '{{guestname}}'; then
|
||
|
||
log "Fallback-vzdump erfolgreich."
|
||
vzdump_success=true
|
||
else
|
||
log "FEHLER: vzdump fehlgeschlagen – auch im Fallback."
|
||
fi
|
||
fi
|
||
|
||
# PBS-Storage ggf. wieder deaktivieren
|
||
if [[ "$vzdump_success" == true && "$pbs_enabled_by_script" == true ]]; then
|
||
log "Deaktiviere temporär aktiviertes PBS-Storage '$BACKUPSTORE' auf $SOURCEHOST..."
|
||
rssh root@"$SOURCEHOST" "pvesm set '$BACKUPSTORE' --disable 1"
|
||
fi
|
||
|
||
# Monitoring-Ausgabe
|
||
if [[ "$vzdump_success" == true ]]; then
|
||
echo "0 DailyPBS - Daily Backup erfolgreich" > /tmp/cmk_tmp.out
|
||
else
|
||
echo "2 DailyPBS - Daily Backup FEHLGESCHLAGEN" > /tmp/cmk_tmp.out
|
||
fi
|
||
|
||
{
|
||
echo "<<<local>>>"
|
||
cat /tmp/cmk_tmp.out
|
||
} > /tmp/90000_checkpbs
|
||
|
||
rscp /tmp/90000_checkpbs root@"$SOURCEHOST":/var/lib/check_mk_agent/spool \
|
||
|| log "Fehler beim Übertragen des Monitoring-Outputs via SCP"
|
||
|
||
rm -f /tmp/cmk_tmp.out /tmp/90000_checkpbs
|
||
#write_pbs_status
|
||
}
|
||
|
||
|
||
run_maintenance() {
|
||
if [[ "${BACKUPSERVER,,}" != "yes" ]]; then
|
||
log "PBS Backup wird übersprungen (BACKUPSERVER=$BACKUPSERVER)"
|
||
return
|
||
fi
|
||
|
||
if [[ "$(date +%u)" == "$MAINTDAY" ]]; then
|
||
log "Running maintenance..."
|
||
PRUNEJOB=$(ssh "$PBSHOST" proxmox-backup-manager prune-job list --output-format json-pretty | grep -m 1 "id" | cut -d'"' -f4)
|
||
ssh root@"$PBSHOST" proxmox-backup-manager prune-job run "$PRUNEJOB"
|
||
ssh root@"$PBSHOST" proxmox-backup-manager garbage-collection start "$BACKUPSTOREPBS"
|
||
ssh root@"$PBSHOST" proxmox-backup-manager verify backup
|
||
else
|
||
log "No maintenance scheduled for today."
|
||
fi
|
||
}
|
||
|
||
run_scrub_stop() {
|
||
local mode="$1" # "local" oder "remote"
|
||
local ssh_cmd=()
|
||
|
||
if [[ "$mode" == "remote" ]]; then
|
||
ssh_cmd=(ssh -p "$SOURCEPORT" root@"$SOURCEHOST")
|
||
fi
|
||
|
||
"${ssh_cmd[@]}" bash -c '
|
||
for pool in $(zpool list -H -o name); do
|
||
echo "Stopping scrub on pool: $pool"
|
||
if zpool status "$pool" | grep -q "scrub in progress"; then
|
||
if zpool scrub -s "$pool"; then
|
||
echo "Scrub stopped on $pool"
|
||
else
|
||
echo "Error stopping scrub on $pool"
|
||
fi
|
||
else
|
||
echo "No active scrub on $pool"
|
||
fi
|
||
done
|
||
'
|
||
}
|
||
|
||
shutdown_now() {
|
||
if [[ "${SHUTDOWN,,}" == "yes" ]]; then
|
||
send_piggyback
|
||
send_piggyback_external
|
||
send_checkzfs_external
|
||
log "Shutting down now..."
|
||
shutdown now
|
||
else
|
||
log "No shutdown requested."
|
||
fi
|
||
}
|
||
|
||
send_piggyback_external() {
|
||
if [[ "${EPIGGYBACK,,}" != "yes" ]]; then
|
||
log "Externer Piggyback-Export deaktiviert."
|
||
return
|
||
fi
|
||
get_sourcehostname
|
||
log "Ermittelter SOURCEHOSTNAME: $SOURCEHOSTNAME"
|
||
|
||
local combined_host="miyagi-${SOURCEHOSTNAME}-$(hostname)"
|
||
local spool_file="90000_${combined_host}_external"
|
||
local temp_dir
|
||
temp_dir=$(mktemp -d)
|
||
|
||
log "Erzeuge temporäre Piggyback-Datei: $temp_dir/$spool_file"
|
||
|
||
{
|
||
echo "<<<<${combined_host}>>>>"
|
||
/usr/bin/check_mk_agent
|
||
echo "<<<<>>>>"
|
||
} > "$temp_dir/$spool_file"
|
||
|
||
if scp -P "$EPIGGYBACK_PORT" "$temp_dir/$spool_file" "$EPIGGYBACK_HOST:/var/lib/check_mk_agent/spool/"; then
|
||
log "Piggyback-Daten erfolgreich an $EPIGGYBACK_HOST gesendet: $spool_file"
|
||
else
|
||
log "Fehler beim Übertragen der Piggyback-Daten an $EPIGGYBACK_HOST"
|
||
rm -rf "$temp_dir"
|
||
return 1
|
||
fi
|
||
|
||
rm -rf "$temp_dir"
|
||
return 0
|
||
}
|
||
|
||
send_checkzfs_external() {
|
||
if [[ "${ECHECKZFS,,}" != "yes" ]]; then
|
||
log "Externer Piggyback-Export deaktiviert."
|
||
return
|
||
fi
|
||
local config="/etc/bashclub/${SOURCEHOST}.conf"
|
||
if [[ ! -f "$config" ]]; then
|
||
log "Konfigurationsdatei fehlt: $config"
|
||
return 1
|
||
fi
|
||
source "$config"
|
||
|
||
if [[ "${ECHECKZFS,,}" != "yes" ]]; then
|
||
log "Externer Check-ZFS deaktiviert (ECHECKZFS=$ECHECKZFS)"
|
||
return
|
||
fi
|
||
|
||
get_sourcehostname # Setzt SOURCEHOSTNAME, wenn nicht vorhanden
|
||
|
||
local checkzfs_cmd="${checkzfs_cmd:-$(command -v checkzfs)}"
|
||
if [[ -z "$checkzfs_cmd" || ! -x "$checkzfs_cmd" ]]; then
|
||
log "checkzfs nicht gefunden oder nicht ausführbar – Abbruch."
|
||
return 1
|
||
fi
|
||
log "Verwende checkzfs: $checkzfs_cmd"
|
||
|
||
# Filter anhand ZFS-Tags auf dem SOURCEHOST aufbauen
|
||
local filter=""
|
||
while IFS=$'\t' read -r name value source; do
|
||
if [[ "$source" == "local" && "$value" == "subvols" ]]; then
|
||
filter+="${name}/|"
|
||
elif [[ "$source" == "local" && "$value" == "all" ]]; then
|
||
filter+="${name}|"
|
||
fi
|
||
done < <(ssh -p "$SOURCEPORT" "$SOURCEHOST" "zfs get -H -o name,value,source -t filesystem,volume $ZPUSHTAG")
|
||
|
||
filter="#${filter%|}"
|
||
log "Generierter ZFS-Filter: $filter"
|
||
|
||
local combined="miyagi-${SOURCEHOSTNAME}-$(hostname)"
|
||
local spoolfile="/tmp/${combined}_checkzfs_external"
|
||
local spooldest="90000_${combined}_checkzfs_external"
|
||
|
||
log "Führe checkzfs aus..."
|
||
{
|
||
echo "<<<local>>>"
|
||
"$checkzfs_cmd" \
|
||
--source root@"${SOURCEHOST}:${SOURCEPORT}" \
|
||
--filter "$filter" \
|
||
--replicafilter "^${ZFSTRGT}" \
|
||
--prefix "$checkzfs_prefix" \
|
||
--threshold "$checkzfs_max_age" \
|
||
--maxsnapshots "$checkzfs_max_snapshot_count" \
|
||
--output checkmk
|
||
} > "$spoolfile"
|
||
|
||
if scp -P "$ECHECKZFS_PORT" "$spoolfile" root@"$ECHECKZFS_HOST":/var/lib/check_mk_agent/spool/"$spooldest"; then
|
||
log "Spool-Datei erfolgreich übertragen an $ECHECKZFS_HOST:$spooldest"
|
||
else
|
||
log "Fehler beim Übertragen der Spool-Datei an $ECHECKZFS_HOST"
|
||
fi
|
||
|
||
rm -f "$spoolfile"
|
||
}
|
||
|
||
# Main execution:
|
||
if [[ $# -eq 0 ]]; then
|
||
if [[ -n "$CONFIG_FILE" ]]; then
|
||
log "Backup-Routine startet in 60 Sekunden..."
|
||
sleep 60
|
||
#log "Starting full backup routine..."
|
||
log "Running full backup using configuration file: $CONFIG_FILE"
|
||
write_zsync_config
|
||
run_zsync
|
||
|
||
if [[ "${BACKUPSERVER,,}" == "yes" ]]; then
|
||
log "BACKUPSERVER ist aktiviert, führe Backup aus..."
|
||
run_maintenance
|
||
run_pbs_backup
|
||
else
|
||
log "BACKUPSERVER ist nicht aktiviert (BACKUPSERVER=$BACKUPSERVER), überspringe Backup."
|
||
fi
|
||
run_remote_updates
|
||
shutdown_now
|
||
else
|
||
usage
|
||
fi
|
||
else
|
||
case "$1" in
|
||
help)
|
||
declare -F | awk '{print $3}' | grep -v "^_" | grep -v "^main$"
|
||
;;
|
||
*)
|
||
if declare -F "$1" >/dev/null 2>&1; then
|
||
"$@"
|
||
else
|
||
log "Function $1 not found."
|
||
usage
|
||
fi
|
||
;;
|
||
esac
|
||
fi
|