This commit is contained in:
2025-07-24 12:52:46 +02:00
parent 849b5e9570
commit a15a7bad54
+196 -206
View File
@@ -25,8 +25,6 @@ while getopts "c:" opt; do
*) usage ;;
esac
done
# Bei Einzelaufruf muss Konfiguration geladen sein
shift $((OPTIND - 1))
if [[ -n "${CONFIG_FILE:-}" ]]; then
@@ -36,110 +34,91 @@ if [[ -n "${CONFIG_FILE:-}" ]]; then
fi
if ! bash -n "$CONFIG_FILE"; then
log "Syntaxfehler in Konfigurationsdatei $CONFIG_FILE"
log "Syntax error in configuration file $CONFIG_FILE"
exit 1
fi
source "$CONFIG_FILE"
# ==========================
# Konfigurationsprüfung
# ==========================
REQUIRED_VARS=(
SOURCEPORT
BACKUPSERVER
ZSYNC
MAINTDAY
SHUTDOWN
UPDATES
SOURCEHOST
ZFSROOT
ZFSSECOND
ZFSTRGT
ZPUSHTAG
ZPUSHMINKEEP
ZPUSHKEEP
ZPUSHLABEL
PBSHOST
BACKUPSTORE
BACKUPSTOREPBS
BACKUPEXCLUDE
REPLEXCLUDE
SOURCEPORT BACKUPSERVER ZSYNC MAINTDAY SHUTDOWN UPDATES
SOURCEHOST ZFSROOT ZFSSECOND ZFSTRGT ZPUSHTAG ZPUSHMINKEEP ZPUSHKEEP ZPUSHLABEL
PBSHOST BACKUPSTORE BACKUPSTOREPBS BACKUPEXCLUDE REPLEXCLUDE
)
MISSING_VARS=()
for var in "${REQUIRED_VARS[@]}"; do
#if [[ -z "${!var:-}" ]]; then
if ! declare -p "$var" &>/dev/null || [[ -z "${!var}" ]]; then
if [[ -z "${!var:-}" ]]; then
MISSING_VARS+=("$var")
fi
done
if [[ "${#MISSING_VARS[@]}" -ne 0 ]]; then
log "Fehlende Konfigurationsvariablen:"
if [[ "${#MISSING_VARS[@]}" -gt 0 ]]; then
log "Missing configuration variables:"
for var in "${MISSING_VARS[@]}"; do
log " - $var"
done
log "Breche ab bitte Konfigurationsdatei prüfen."
log "Aborting — please check your configuration file."
exit 1
else
log "Alle erforderlichen Konfigurationsvariablen sind gesetzt."
log "All required configuration variables are set."
fi
fi
# Funktionen
#remote_ssh() {
# ssh -p "$SOURCEPORT" "$@"
#}
rssh() { ssh -p "$SOURCEPORT" "$@"; }
rscp() { scp -P "$SOURCEPORT" "$@"; }
#remote_scp() {
# scp -P "$SOURCEPORT" "$@"
#}
get_sourcehostname() {
if [[ -z "${SOURCEHOSTNAME:-}" ]]; then
log "SOURCEHOSTNAME is empty, retrieving via SSH from $SOURCEHOST..."
SOURCEHOSTNAME=$(rssh "$SOURCEHOST" hostname)
log "Detected SOURCEHOSTNAME: $SOURCEHOSTNAME"
fi
}
set_wol_g_enabled() {
log "Pruefe, ob ethtool installiert ist..."
log "Checking if ethtool is installed..."
if ! command -v ethtool >/dev/null 2>&1; then
log "ethtool ist nicht installiert, versuche Installation..."
log "ethtool is not installed, attempting installation..."
apt update && apt install -y ethtool || {
log "Fehler: ethtool konnte nicht installiert werden."
log "Error: ethtool could not be installed."
return 1
}
else
log "ethtool ist bereits installiert."
log "ethtool is already installed."
fi
log "Pruefe und setze Wake-on-LAN (WOL) auf 'g' nur bei Interfaces mit statischer IP..."
log "Setting Wake-on-LAN (WOL) to 'g' on interfaces with static IP..."
for iface in $(ls /sys/class/net | grep -vE '^(lo|tap|vmbr|veth|br|docker|bond|wl)'); do
if [[ -e "/sys/class/net/$iface/device" ]]; then
log "Bearbeite physisches Interface: $iface"
log "Processing physical interface: $iface"
# Aktuellen WOL-Status pruefen
current_wol=$(ethtool "$iface" 2>/dev/null | awk '/Wake-on:/ {print $2}')
if [[ "$current_wol" != "g" ]]; then
log "Setze WOL auf 'g' fuer $iface..."
ethtool -s "$iface" wol g || log "Fehler beim Setzen von WOL auf $iface"
log "Setting WOL to 'g' for $iface..."
ethtool -s "$iface" wol g || log "Error setting WOL on $iface"
else
log "WOL ist bereits korrekt auf 'g' fuer $iface"
log "WOL already set to 'g' for $iface"
fi
# Pruefen, ob ein 'iface $iface inet static' Eintrag existiert
if grep -qE "^\s*iface\s+$iface\s+inet\s+static" /etc/network/interfaces; then
if ! grep -A 5 -E "^\s*iface\s+$iface\s+inet\s+static" /etc/network/interfaces | grep -q "post-up ethtool -s $iface wol g"; then
log "Ergaenze WOL-Befehl im statischen Block fuer $iface..."
log "Adding WOL command in static block for $iface..."
sed -i "/^\s*iface\s\+$iface\s\+inet\s\+static/a \ post-up ethtool -s $iface wol g" /etc/network/interfaces
else
log "WOL-Befehl fuer $iface ist bereits im statischen Block vorhanden."
log "WOL command already present in static block for $iface."
fi
else
log "Kein statischer Eintrag fuer $iface gefunden, keine Aenderung vorgenommen."
log "No static entry found for $iface, no changes made."
fi
fi
done
}
write_zsync_config() {
get_sourcehostname
local conf_file="/etc/bashclub/$SOURCEHOST.conf"
log "Writing zsync config to $conf_file"
{
@@ -166,30 +145,34 @@ run_zsync() {
if [[ "$ZSYNC" != "no" ]]; then
/usr/bin/bashclub-zsync -c "/etc/bashclub/$SOURCEHOST.conf"
else
log "Zsync is disabled"
log "Zsync is disabled."
fi
}
run_remote_updates() {
if [[ "$UPDATES" == "yes" ]]; then
ssh "$PBSHOST" apt update && apt dist-upgrade -y
if [[ "${UPDATES,,}" == "yes" ]]; then
log "Running updates on local system..."
apt update && apt dist-upgrade -y || log "Error during local updates"
if [[ "${BACKUPSERVER,,}" == "yes" ]]; then
log "Running updates on PBS host ($PBSHOST)..."
ssh root@"$PBSHOST" apt update && ssh root@"$PBSHOST" apt dist-upgrade -y || {
log "Error during updates on $PBSHOST"
}
else
log "Remote updates disabled"
log "PBS updates skipped (BACKUPSERVER=$BACKUPSERVER)"
fi
else
log "Updates disabled (UPDATES=$UPDATES)"
fi
}
send_piggyback_data() {
# Falls SOURCEHOSTNAME leer ist, ueber SSH vom Zielhost ermitteln
if [[ -z "${SOURCEHOSTNAME:-}" ]]; then
log "SOURCEHOSTNAME is empty retrieving via SSH from $SOURCEHOST..."
SOURCEHOSTNAME=$(ssh -p "$SOURCEPORT" "$SOURCEHOST" hostname)
log "Detected SOURCEHOSTNAME: $SOURCEHOSTNAME"
fi
send_piggyback() {
get_sourcehostname
local combined_host="miyagi-${SOURCEHOSTNAME}-$(hostname)"
local filename="90000_${combined_host}"
log "Do nott forget to add a Host in CMK named: ${combined_host} (without Agent, Piggyback enabled)!"
log "Reminder: Add a host named ${combined_host} in CMK (without Agent, Piggyback enabled)!"
log "Creating piggyback file: $filename"
{
@@ -198,76 +181,94 @@ send_piggyback_data() {
echo "<<<<>>>>"
} > "$filename"
if scp -P "$SOURCEPORT" "$filename" "$SOURCEHOST:/var/lib/check_mk_agent/spool/"; then
if rscp "$filename" "$SOURCEHOST:/var/lib/check_mk_agent/spool/"; then
log "Piggyback data successfully sent to $SOURCEHOST"
else
log "ERROR: Failed to send piggyback data to $SOURCEHOST"
fi
rm -f "$filename"
}
run_pbs_backup() {
if [[ -z "${SOURCEHOSTNAME:-}" ]]; then
SOURCEHOSTNAME=$(ssh -p "$SOURCEPORT" root@"$SOURCEHOST" hostname)
if [[ "${BACKUPSERVER,,}" != "yes" ]]; then
log "PBS Backup übersprungen: BACKUPSERVER='$BACKUPSERVER' (muss 'yes' sein)"
return 0
fi
log "Running PBS vzdump job..."
get_sourcehostname
log "Starte PBS Backup auf Host: $SOURCEHOST"
# PBS-Storage ggf. aktivieren
log "Checking if PBS storage '$BACKUPSTORE' is enabled on $SOURCEHOST..."
if ssh -p "$SOURCEPORT" root@"$SOURCEHOST" "pvesm status | grep -w '$BACKUPSTORE' | grep -q 'disabled'"; then
log "PBS storage '$BACKUPSTORE' is disabled. Attempting to enable... sleep 10 Sekunden"
ssh -p "$SOURCEPORT" root@"$SOURCEHOST" "pvesm set '$BACKUPSTORE' --disable 0 && sleep 10"
# PBS-Storage prüfen und ggf. aktivieren
log "Prüfe, ob PBS-Storage '$BACKUPSTORE' auf $SOURCEHOST aktiviert ist..."
if rssh root@"$SOURCEHOST" "pvesm status | grep -w '$BACKUPSTORE' | grep -q 'disabled'"; then
log "PBS-Storage '$BACKUPSTORE' ist deaktiviert aktiviere temporär und warte 10 Sekunden..."
rssh root@"$SOURCEHOST" "pvesm set '$BACKUPSTORE' --disable 0 && sleep 10"
pbs_enabled_by_script=true
else
log "PBS storage '$BACKUPSTORE' is already enabled."
log "PBS-Storage '$BACKUPSTORE' ist bereits aktiviert."
pbs_enabled_by_script=false
fi
vzdump_success=false
# Hauptversuch mit --pbs-change-detection-mode
if ssh -p "$SOURCEPORT" root@"$SOURCEHOST" vzdump --pbs-change-detection-mode metadata \
# Versuch: vzdump mit Change Detection
## Eintrag muss in /etc/vzdump.conf entries-max 100000000
log "Starte vzdump mit Change Detection..."
if rssh root@"$SOURCEHOST" vzdump --pbs-change-detection-mode metadata \
--node "$SOURCEHOSTNAME" --storage "$BACKUPSTORE" \
--exclude "$BACKUPEXCLUDE" --mode snapshot --all 1 \
--notes-template '{{guestname}}'; then
log "vzdump with change-detection-mode succeeded"
log "vzdump (Change Detection) erfolgreich."
vzdump_success=true
else
log "Fallback: vzdump with change-detection-mode failed, trying without it..."
log "vzdump (Change Detection) fehlgeschlagen versuche Fallback ohne Change Detection..."
if ssh -p "$SOURCEPORT" root@"$SOURCEHOST" vzdump \
if rssh root@"$SOURCEHOST" vzdump \
--node "$SOURCEHOSTNAME" --storage "$BACKUPSTORE" \
--exclude "$BACKUPEXCLUDE" --mode snapshot --all 1 \
--notes-template '{{guestname}}'; then
log "Fallback vzdump succeeded"
log "Fallback-vzdump erfolgreich."
vzdump_success=true
else
log "ERROR: vzdump failed even after fallback"
log "FEHLER: vzdump fehlgeschlagen auch im Fallback."
fi
fi
# PBS-Storage wieder deaktivieren, wenn zuvor aktiviert und erfolgreich
# PBS-Storage ggf. wieder deaktivieren
if [[ "$vzdump_success" == true && "$pbs_enabled_by_script" == true ]]; then
log "Disabling PBS storage '$BACKUPSTORE' again on $SOURCEHOST..."
ssh -p "$SOURCEPORT" root@"$SOURCEHOST" "pvesm set '$BACKUPSTORE' --disable 1"
log "Deaktiviere temporär aktiviertes PBS-Storage '$BACKUPSTORE' auf $SOURCEHOST..."
rssh root@"$SOURCEHOST" "pvesm set '$BACKUPSTORE' --disable 1"
fi
# Monitoring-Output für Checkmk
# Monitoring-Ausgabe
if [[ "$vzdump_success" == true ]]; then
echo "0 DailyPBS - Daily Backup" > /tmp/cmk_tmp.out
echo "0 DailyPBS - Daily Backup erfolgreich" > /tmp/cmk_tmp.out
else
echo "2 DailyPBS - Daily Backup FAILED" > /tmp/cmk_tmp.out
echo "2 DailyPBS - Daily Backup FEHLGESCHLAGEN" > /tmp/cmk_tmp.out
fi
( echo "<<<local>>>" ; cat /tmp/cmk_tmp.out ) > /tmp/90000_checkpbs
scp -P "$SOURCEPORT" /tmp/90000_checkpbs root@"$SOURCEHOST":/var/lib/check_mk_agent/spool || log "Fehler beim SCP des Monitoring-Outputs"
{
echo "<<<local>>>"
cat /tmp/cmk_tmp.out
} > /tmp/90000_checkpbs
rscp /tmp/90000_checkpbs root@"$SOURCEHOST":/var/lib/check_mk_agent/spool \
|| log "Fehler beim Übertragen des Monitoring-Outputs via SCP"
rm -f /tmp/cmk_tmp.out /tmp/90000_checkpbs
#write_pbs_status
}
run_maintenance() {
if [[ "${BACKUPSERVER,,}" != "yes" ]]; then
log "PBS Backup wird übersprungen (BACKUPSERVER=$BACKUPSERVER)"
return
fi
if [[ "$(date +%u)" == "$MAINTDAY" ]]; then
log "Running maintenance..."
PRUNEJOB=$(ssh "$PBSHOST" proxmox-backup-manager prune-job list --output-format json-pretty | grep -m 1 "id" | cut -d'"' -f4)
@@ -279,42 +280,35 @@ run_maintenance() {
fi
}
run_scrub_stop_src() {
ssh -p "$SOURCEPORT" root@"$SOURCEHOST" 'for pool in $(zpool list -H -o name); do
echo "Stoppe Scrub auf Pool: $pool"
if zpool status "$pool" | grep -q "scrub in progress"; then
if zpool scrub -s "$pool"; then
echo "Scrub auf $pool gestoppt"
else
echo "Fehler beim Stoppen des Scrubs auf $pool"
fi
else
echo "Kein aktiver Scrub auf $pool"
fi
done'
}
run_scrub_stop() {
local mode="$1" # "local" oder "remote"
local ssh_cmd=()
run_scrub_stop_local() {
if [[ "$mode" == "remote" ]]; then
ssh_cmd=(ssh -p "$SOURCEPORT" root@"$SOURCEHOST")
fi
"${ssh_cmd[@]}" bash -c '
for pool in $(zpool list -H -o name); do
echo "Stoppe Scrub auf Pool: $pool"
echo "Stopping scrub on pool: $pool"
if zpool status "$pool" | grep -q "scrub in progress"; then
if zpool scrub -s "$pool"; then
echo "Scrub auf $pool gestoppt"
echo "Scrub stopped on $pool"
else
echo "Fehler beim Stoppen des Scrubs auf $pool"
echo "Error stopping scrub on $pool"
fi
else
echo "Kein aktiver Scrub auf $pool"
echo "No active scrub on $pool"
fi
done
'
}
shutdown_if_requested() {
log "SHUTDOWN-Variable: '${SHUTDOWN:-nicht gesetzt}'"
#if [[ "${SHUTDOWN,,}" == "yes" ]]; then
if [[ "$(echo "$SHUTDOWN" | tr '[:upper:]' '[:lower:]')" == "yes" ]]; then
send_piggyback_data
send_piggyback_data_external
shutdown_now() {
if [[ "${SHUTDOWN,,}" == "yes" ]]; then
send_piggyback
send_piggyback_external
send_checkzfs_external
log "Shutting down now..."
shutdown now
else
@@ -322,117 +316,113 @@ shutdown_if_requested() {
fi
}
send_piggyback_data_external() {
send_piggyback_external() {
if [[ "${EPIGGYBACK,,}" != "yes" ]]; then
log "Externer Piggyback-Export deaktiviert."
return
fi
if [[ -z "$EPIGGYBACK_HOST" || -z "$EPIGGYBACK_PORT" ]]; then
log "EPIGGYBACK_HOST oder EPIGGYBACK_PORT nicht gesetzt Abbruch."
return 1
fi
get_sourcehostname
log "Ermittelter SOURCEHOSTNAME: $SOURCEHOSTNAME"
local combined_host="miyagi-${SOURCEHOSTNAME}-$(hostname)"
local filename="90000_${combined_host}_external"
local spool_file="90000_${combined_host}_external"
local temp_dir
temp_dir=$(mktemp -d)
log "Erzeuge externe Piggyback-Datei: $filename"
log "Erzeuge temporäre Piggyback-Datei: $temp_dir/$spool_file"
{
echo "<<<<${combined_host}>>>>"
/usr/bin/check_mk_agent
echo "<<<<>>>>"
} > "$filename"
} > "$temp_dir/$spool_file"
if scp -P "$EPIGGYBACK_PORT" "$filename" "$EPIGGYBACK_HOST:/var/lib/check_mk_agent/spool/"; then
log "Piggyback-Daten erfolgreich an $EPIGGYBACK_HOST gesendet."
if scp -P "$EPIGGYBACK_PORT" "$temp_dir/$spool_file" "$EPIGGYBACK_HOST:/var/lib/check_mk_agent/spool/"; then
log "Piggyback-Daten erfolgreich an $EPIGGYBACK_HOST gesendet: $spool_file"
else
log "Fehler beim Senden der Piggyback-Daten an $EPIGGYBACK_HOST"
log "Fehler beim Übertragen der Piggyback-Daten an $EPIGGYBACK_HOST"
rm -rf "$temp_dir"
return 1
fi
rm -f "$filename"
rm -rf "$temp_dir"
return 0
}
send_checkzfs_output_external() {
if [[ "${ECHECKZFS,,}" == "yes" ]]; then
log "ECHECKZFS aktiviert beginne Monitoring-Ausgabe."
else
log "ECHECKZFS ist deaktiviert überspringe Monitoring-Ausgabe."
send_checkzfs_external() {
if [[ "${ECHECKZFS,,}" != "yes" ]]; then
log "Externer Piggyback-Export deaktiviert."
return
fi
local config="/etc/bashclub/${SOURCEHOST}.conf"
if [[ ! -f "$config" ]]; then
log "Konfigurationsdatei fehlt: $config"
return 1
fi
source "$config"
if [[ "${ECHECKZFS,,}" != "yes" ]]; then
log "Externer Check-ZFS deaktiviert (ECHECKZFS=$ECHECKZFS)"
return
fi
local checkzfs_cmd
checkzfs_cmd=$(which checkzfs)
get_sourcehostname # Setzt SOURCEHOSTNAME, wenn nicht vorhanden
if [[ -z "$checkzfs_cmd" ]]; then
log "check_zfs ist nicht installiert Abbruch."
return
local checkzfs_cmd="${checkzfs_cmd:-$(command -v checkzfs)}"
if [[ -z "$checkzfs_cmd" || ! -x "$checkzfs_cmd" ]]; then
log "checkzfs nicht gefunden oder nicht ausführbar Abbruch."
return 1
fi
log "Verwende checkzfs: $checkzfs_cmd"
local config_file="/etc/bashclub/${SOURCEHOST}.conf"
if [[ ! -f "$config_file" ]]; then
log "Konfigurationsdatei $config_file nicht gefunden check_zfs-Ausführung abgebrochen."
return
# Filter anhand ZFS-Tags auf dem SOURCEHOST aufbauen
local filter=""
while IFS=$'\t' read -r name value source; do
if [[ "$source" == "local" && "$value" == "subvols" ]]; then
filter+="${name}/|"
elif [[ "$source" == "local" && "$value" == "all" ]]; then
filter+="${name}|"
fi
done < <(ssh -p "$SOURCEPORT" "$SOURCEHOST" "zfs get -H -o name,value,source -t filesystem,volume $ZPUSHTAG")
# Konfiguration einlesen
log "Lese Konfiguration aus $config_file..."
source "$config_file"
filter="#${filter%|}"
log "Generierter ZFS-Filter: $filter"
# Prüfung auf deaktiviertes Monitoring in der Konfig
if [[ "${checkzfs_disabled:-0}" -eq 1 ]]; then
log "check_zfs Monitoring laut Konfiguration deaktiviert überspringe."
return
fi
local combined="miyagi-${SOURCEHOSTNAME}-$(hostname)"
local spoolfile="/tmp/${combined}_checkzfs_external"
local spooldest="90000_${combined}_checkzfs_external"
local spool_tmp="/tmp/${checkzfs_spool_maxage}_${checkzfs_prefix}"
local remote_host="$ECHECKZFS_HOST"
local remote_port="$ECHECKZFS_PORT"
echo "<<<local>>>" > "$spool_tmp"
log "Generiere check_zfs Monitoring-Ausgabe mit Prefix $checkzfs_prefix..."
$checkzfs_cmd --source "$source:$soru" \
--output checkmk \
log "Führe checkzfs aus..."
{
echo "<<<local>>>"
"$checkzfs_cmd" \
--source root@"${SOURCEHOST}:${SOURCEPORT}" \
--filter "$filter" \
--replicafilter "^${ZFSTRGT}" \
--prefix "$checkzfs_prefix" \
--threshold "$checkzfs_max_age" \
--maxsnapshots "$checkzfs_max_snapshot_count" \
--prefix "$checkzfs_prefix" \
--replicafilter "^${target}" \
--filter "$snapshot_filter" >> "$spool_tmp"
--output checkmk
} > "$spoolfile"
if [[ -s "$spool_tmp" ]]; then
log "Übertrage Spool-Datei an $remote_host über Port $remote_port..."
scp -P "$remote_port" "$spool_tmp" root@"$remote_host":/var/lib/check_mk_agent/spool/ && \
log "Spool-Datei erfolgreich übertragen." || \
log "Fehler beim Übertragen der Spool-Datei."
rm -f "$spool_tmp"
if scp -P "$ECHECKZFS_PORT" "$spoolfile" root@"$ECHECKZFS_HOST":/var/lib/check_mk_agent/spool/"$spooldest"; then
log "Spool-Datei erfolgreich übertragen an $ECHECKZFS_HOST:$spooldest"
else
log "check_zfs-Ausgabe war leer keine Übertragung erfolgt."
rm -f "$spool_tmp"
log "Fehler beim Übertragen der Spool-Datei an $ECHECKZFS_HOST"
fi
rm -f "$spoolfile"
}
main() {
# Main execution:
if [[ $# -eq 0 ]]; then
if [[ -n "$CONFIG_FILE" ]]; then
log "Backup-Routine startet in 60 Sekunden..."
sleep 60
log "Starting full backup routine..."
log "Versuche Hostname von $SOURCEHOST via SSH (Port $SOURCEPORT) abzurufen..."
if ! SOURCEHOSTNAME=$(ssh -p "$SOURCEPORT" -o ConnectTimeout=5 -o BatchMode=yes "$SOURCEHOST" hostname 2>/dev/null); then
log "Error: Hostname von $SOURCEHOST konnte nicht abgerufen werden Abbruch."
exit 1
fi
log "Hostname von $SOURCEHOST: $SOURCEHOSTNAME"
set_wol_g_enabled
#log "Starting full backup routine..."
log "Running full backup using configuration file: $CONFIG_FILE"
write_zsync_config
run_zsync
send_checkzfs_output_external
run_scrub_stop_local
run_scrub_stop_src
if [[ "${BACKUPSERVER,,}" == "yes" ]]; then
log "BACKUPSERVER ist aktiviert, führe Backup aus..."
@@ -441,23 +431,23 @@ main() {
else
log "BACKUPSERVER ist nicht aktiviert (BACKUPSERVER=$BACKUPSERVER), überspringe Backup."
fi
run_remote_updates
# run_updates
shutdown_if_requested
}
# Funktionsbasierter Aufruf
if [[ "${1:-}" == "help" ]]; then
echo "Verfuegbare Funktionen:"
declare -F | awk '{print " - " $3}' | grep -v "^ - _"
exit 0
elif [[ "${1:-}" =~ ^[a-zA-Z0-9_]+$ && "$(type -t "$1")" == "function" ]]; then
FUNC="$1"
shift
"$FUNC" "$@"
exit 0
shutdown_now
else
main
usage
fi
else
case "$1" in
help)
declare -F | awk '{print $3}' | grep -v "^_" | grep -v "^main$"
;;
*)
if declare -F "$1" >/dev/null 2>&1; then
"$@"
else
log "Function $1 not found."
usage
fi
;;
esac
fi