diff --git a/bashclub-zfs b/bashclub-zfs index c6aed3a..89ad093 100644 --- a/bashclub-zfs +++ b/bashclub-zfs @@ -2,74 +2,135 @@ # backup-zfs: use zfs send/recv to push/pull snapshots - New does not run twice prog="$(basename "$0")" +# Function to display usage usage() { - cat >&2 <<-EOF - usage: $prog [-hvq] [-t tag] [-k keep] [-d dateopts] src dest - use zfs send/recv to push/pull snapshots + cat >&2 <<-EOF + usage: $prog [-hvq] [-t tag] [-k keep] [-d dateopts] src dest + use zfs send/recv to push/pull snapshots - !This Release is default set to send raw. This means you can send encrypted and compressed Datasets 1:1! + !This Release is default set to send raw. This means you can send encrypted and compressed Datasets 1:1! - src the source fs, specified as [host:]pool/path/to/fs - dest the destination fs parent, specified as [host:]pool/path/to/fs - (the final path component of src will be appended to dest) - -p ssh port - -h help - -v verbose mode - -C SSH Compression - obsolete if send_opts="-v -w" are set to raw - -R ZFS Re-Init - -I Send intermediate Snapshots - -q quiet mode - -t tag tag to use for naming snapshots (default: backup-zfs) - -k keep number of snapshots to keep on src (default: 5) - -d dateopts options for date(1) - used to name the snapshots (default: +%F_%T) - -s store mode - output snaps from local fs to ssh server - -r read mode - read snaps from ssh server to local fs - -g gpg-id gpg recipient key id (store mode only) - EOF - exit $1 + src the source fs, specified as [host:]pool/path/to/fs + dest the destination fs parent, specified as [host:]pool/path/to/fs + (the final path component of src will be appended to dest) + -p ssh port + -h help + -v verbose mode + -C SSH Compression - obsolete if send_opts="-v -w" are set to raw + -R ZFS Re-Init + -I Send intermediate Snapshots + -q quiet mode + -t tag tag to use for naming snapshots (default: backup-zfs) + -k keep number of snapshots to keep on src (default: 5) + -d dateopts options for date(1) - used to name the snapshots (default: +%F_%T) + -s store mode - output snaps from local fs to ssh server + -r read mode - read snaps from ssh server to local fs + -g gpg-id gpg recipient key id (store mode only) + EOF + exit $1 } -# log to syslog; if verbose or on a tty, also to stdout -# usage: log msg +# Log function for syslog and stdout (if verbose or tty) log() { - logger -t "$prog" -- "$@" - if ! $quiet && [[ -t 1 ]] || $verbose ; then - echo "$@" >&2 - fi + logger -t "$prog" -- "$@" + if ! $quiet && [[ -t 1 ]] || $verbose ; then + echo "$@" >&2 + fi } -# exit with a code & message -# usage: die $exitcode msg +# Exit with a code & message die() { - code="$1" - shift - if [[ $code -ne 0 ]] ; then - verbose=true log "FATAL: $@" - else - log "$@" - fi - exit $code + code="$1" + shift + if [[ $code -ne 0 ]] ; then + verbose=true log "FATAL: $@" + else + log "$@" + fi + exit $code } -# run zfs(1) command either locally or via ssh -# usage: ZFS "$host" command args... +# Run zfs(1) command either locally or via ssh ZFS() { - host="$1" - shift - if [[ -n $host ]] ; then - log "remote ($host): zfs $@" - ssh $sshcompress -p $port "$host" zfs "$@" - else - log "local: zfs $@" - zfs "$@" - fi + host="$1" + shift + if [[ -n $host ]] ; then + log "remote ($host): zfs $@" + ssh $sshcompress -p $port "$host" zfs "$@" + else + log "local: zfs $@" + zfs "$@" + fi } +# Create a snapshot on the specified filesystem +create_snapshot() { + local host=$1 + local fs=$2 + local snapshot_name=$3 + ZFS "$host" snapshot -r "$fs@$snapshot_name" || die $? "zfs snapshot failed" +} + +# Send or receive ZFS snapshot between source and destination +ZFS_send_receive() { + local src=$1 + local dest=$2 + local options=$3 + local action=$4 + + log "$action: $src to $dest" + if [[ $action == "send" ]]; then + ZFS "$srchost" send $options "$src" | ZFS "$desthost" receive $options -Fue "$dest" || die $? "zfs send failed" + elif [[ $action == "receive" ]]; then + ZFS "$srchost" send $options "$src" | ZFS "$desthost" receive $options -Fue "$dest" || die $? "zfs receive failed" + fi +} + +# Encrypt the snapshot using GPG +encrypt_snapshot() { + local gpg_id=$1 + local snapshot_file=$2 + gpg --trust-model always --encrypt --recipient "$gpg_id" "$snapshot_file" || die $? "GPG encryption failed" +} + +# Cleanup old snapshots on the source filesystem +cleanup_old_snapshots() { + local fs=$1 + local keep=$2 + local tag=$3 + + # Get all snapshots for the dataset + snapshots=$(ZFS "" list -d 1 -t snapshot -H -o name "$fs" | grep -F "@${tag}_" | cut -f2 -d@) + # Calculate the snapshots to keep + old_snapshots=$(echo "$snapshots" | tail -n +$((keep+1))) + + # Delete the old snapshots + for snap in $old_snapshots; do + ZFS "" destroy -r "$fs@$snap" || die $? "zfs snapshot cleanup failed" + done +} + +# Locking function to prevent multiple instances +lock() { + local retries=0 + while ! flock -n 9; do + retries=$((retries + 1)) + if [[ $retries -ge 5 ]]; then + die 1 "Unable to acquire lock after 5 attempts" + fi + sleep 1 + done +} + +unlock() { + flock -u 9 +} + +# Main processing ( - flock -n 9 || exit 1 - ### - ### defaults - ### + flock -n 9 || exit 1 + + # Defaults tag="$prog" dateopts="+%F_%T" keep=5 @@ -81,17 +142,12 @@ ZFS() { sshcompress="" reinit="" intermediate="-i " - ### - ### parse options - ### + + # Parse options while getopts "hvqk:p:t:d:srCIRg:" opt ; do case $opt in h) usage 0 ;; - v) - verbose=true - send_opts="-v -w" - recv_opts="-v" - ;; + v) verbose=true; send_opts="-v -w"; recv_opts="-v" ;; q) quiet=true ;; k) keep=$OPTARG ;; p) port=$OPTARG ;; @@ -99,125 +155,52 @@ ZFS() { d) dateopts=$OPTARG ;; s) tossh=true ;; r) fromssh=true ;; - C) sshcompress="-C " ;; - R) reinit="-R " ;; - I) intermediate="-I " ;; + C) sshcompress="-C" ;; + R) reinit="-R" ;; + I) intermediate="-I" ;; g) gpgid="$OPTARG" ;; *) usage 1 ;; esac done shift $((OPTIND-1)) - date="$(date $dateopts)" - $tossh && $fromssh && die 1 "-s and -r are mutually exclusive" + + # Validate mode conflicts + if $tossh && $fromssh; then + die 1 "-s and -r are mutually exclusive" + fi if ! $tossh && [[ -n $gpgid ]] ; then die 1 "-g can only be used with -s" fi - ### - ### parse src & dest host/fs info - ### - # fail if there's ever >1 colon - if [[ $1 =~ :.*: || $2 =~ :.*: ]] ; then + # Parse src & dest host/fs info + if [[ $1 =~ :.*: || $2 =~ :.*: ]]; then die 1 "invalid fsspec: '$1' or '$2'" fi - - # fail if src or dest isn't specified - if [[ -z $1 || -z $2 ]] ; then + if [[ -z $1 || -z $2 ]]; then usage 1 fi src="$1" dest="$2" - ### - ### ssh mode - output snaps from local fs to ssh or read snaps from ssh to local fs - if $tossh ; then + # SSH mode (store or read) + if $tossh; then log "sending from local zfs filesystem to SSH server" + create_snapshot "$srchost" "$srcfs" "$snap" - # make sure src exists - if [[ $src =~ : ]] ; then - die 1 "$src must be a local zfs filesystem" - elif [[ $(ZFS "" list -H -o name "$src" 2>/dev/null) != $src ]] ; then - die 1 "$src must be a local zfs filesystem" + # Incremental send if applicable + last_snapshot="$(ssh "$desthost" zfslast)" + if [[ -z $last_snapshot ]]; then + die 1 "No valid incremental path from $src to $dest" fi - # split dest to components - if [[ $dest =~ : ]] ; then - desthost="${dest%:*}" - destpath="${dest#*:}" - else - die 1 "$dest must be ssh host:path" - fi - - # get the last src component - srcbase="${src##*/}" - - ### - ### create new snapshot on src - ### - snap="${tag}_$date" - cur="$src@$snap" - ZFS "$srchost" snapshot -r "$cur" || die $? "zfs snapshot failed" - - ### - ### get newest snapshot on dest - it must exist on src - ### - #last="$(ZFS "$desthost" list -d 1 -t snapshot -H -S creation -o name $destfs/$srcbase | head -n1 | cut -f2 -d@)" - last="$(ssh "$desthost" zfslast)" - - ### - ### send - ### - # refuse to send without a valid .last maker - if [[ -z $last ]] ; then - die 1 "ssh path contains no .last file" - # special case: tagged snapshots exist on dest, but src has rotated through all - elif ! ZFS "$srchost" list $src@$last &>/dev/null ; then - die 1 "no incremental path from from $src to $dest" - # normal case: send incremental - else - log "sending $([[ -n $gpgid ]] && echo "encrypted ")incremental snapshot from $src to $dest (${last#${tag}_}..${cur#*@${tag}_})" - #ZFS "$srchost" send $send_opts -R $intermediate "$last" "$cur" | ZFS "$desthost" receive $recv_opts -Fue "$destfs" || die $? "zfs incremental send failed" - if [[ -n $gpgid ]] ; then - ZFS "$srchost" send $send_opts -R $intermediate "$last" "$cur" \ - | gpg --trust-model always --encrypt --recipient "$gpgid" \ - | ssh "$desthost" zfswrite "${tag}_$date.zfssnap.gpg" \ - || die $? "zfs incremental send failed" - ssh "$desthost" zfslast "$snap" - else - ZFS "$srchost" send $send_opts -R $intermediate "$last" "$cur" \ - | ssh "$desthost" zfswrite "${tag}_$date.zfssnap" \ - || die $? "zfs incremental send failed" - ssh "$desthost" zfslast "$snap" - fi - fi + ZFS_send_receive "$last_snapshot" "$cur" "$send_opts" "send" exit - elif $fromssh ; then - log "receving from SSH server to local zfs filesystem" - - # make sure dest exists - if [[ $dest =~ : ]] ; then - die 1 "$dest must be a local zfs filesystem" - elif [[ $(ZFS "" list -H -o name "$dest" 2>/dev/null) != $dest ]] ; then - die 1 "$dest must be a local zfs filesystem" - fi - - # split src into components - if [[ $src =~ : ]] ; then - srchost="${src%:*}" - srcpath="${src#*:}" - else - die 1 "$src must be ssh host:path" - fi - - ### - ### receive - ### - log "receiving incremental snapshot from $src to $dest" - #ZFS "$srchost" send $send_opts -R $intermediate "$last" "$cur" | ZFS "$desthost" receive $recv_opts -Fue "$destfs" || die $? "zfs incremental send failed" - for file in $(ssh "$srchost" zfsfind | sort) ; do + elif $fromssh; then + log "receiving from SSH server to local zfs filesystem" + for file in $(ssh "$srchost" zfsfind | sort); do log "receiving $file from $srchost" - if [[ $file =~ \.gpg$ ]] ; then + if [[ $file =~ \.gpg$ ]]; then ssh "$srchost" zfsget "$file" | gpg | ZFS "$desthost" receive $recv_opts -Fue "$dest" \ && ssh "$srchost" rm "$file" else @@ -225,60 +208,20 @@ ZFS() { && ssh "$srchost" rm "$file" fi done - exit fi - # discard anything before a colon to get the fs - srcfs="${src#*:}" - destfs="${dest#*:}" - - # iff there is a colon, discard everything after it to get the host - [[ $src =~ : ]] && srchost="${src%:*}" - [[ $dest =~ : ]] && desthost="${dest%:*}" - - # get the last src component - srcbase="${srcfs##*/}" - - # ensure the destination fs exists before proceeding - if [[ $(ZFS "$desthost" list -H -o name "$destfs" 2>/dev/null) != $destfs ]] ; then - die 1 "destination fs '$destfs' doesn't exist" - fi - - ### - ### create new snapshot on src - ### + # Main snapshot creation and transfer process cur="$srcfs@${tag}_$date" - ZFS "$srchost" snapshot -r "$cur" || die $? "zfs snapshot failed" + create_snapshot "$srchost" "$srcfs" "$cur" - ### - ### get newest snapshot on dest - it must exist on src - ### - last="$(ZFS "$desthost" list -d 1 -t snapshot -H -S creation -o name $destfs/$srcbase | head -n1 | cut -f2 -d@)" - - ### - ### send & receive - ### - # 1st time: send full snapshot - if [[ -z $last ]] ; then - log "sending full recursive snapshot from $src to $dest" - ZFS "$srchost" send $send_opts $reinit "$cur" | ZFS "$desthost" receive $recv_opts -Fue "$destfs" || die $? "zfs full send failed" - # special case: tagged snapshots exist on dest, but src has rotated through all - elif ! ZFS "$srchost" list $srcfs@$last &>/dev/null ; then - die 1 "no incremental path from from $src to $dest" - # normal case: send incremental + last_snapshot="$(ZFS "$desthost" list -d 1 -t snapshot -H -S creation -o name "$destfs" | head -n 1)" + if [[ -z $last_snapshot ]]; then + ZFS_send_receive "$cur" "$destfs" "$send_opts" "send" else - log "sending incremental snapshot from $src to $dest (${last#${tag}_}..${cur#*@${tag}_})" - ZFS "$srchost" send $send_opts -R $intermediate "$last" "$cur" | ZFS "$desthost" receive $recv_opts -Fue "$destfs" || die $? "zfs incremental send failed" + ZFS_send_receive "$last_snapshot" "$cur" "$send_opts" "send" fi - ### - ### clean up old snapshots - ### - for snap in $(ZFS "$srchost" list -d 1 -t snapshot -H -S creation -o name $srcfs \ - | grep -F "@${tag}_" | cut -f2 -d@ | tail -n+$((keep+1)) ) ; - do - ZFS "$srchost" destroy -r $srcfs@$snap - done - + # Clean up old snapshots on source + cleanup_old_snapshots "$srcfs" "$keep" "$tag" ) 9>/var/lock/bashclub-zfs.lock