From bc90f87636f065bfdec6018eb83263dcb3508c08 Mon Sep 17 00:00:00 2001 From: Alice Gaudon Date: Fri, 13 Nov 2020 11:06:19 +0100 Subject: [PATCH] Make zfs backups resumable and more improvements --- zfs_backup.sh | 34 +++++++++------- zfs_backup_dataset.sh | 94 ++++++++++++++++++++++++++----------------- 2 files changed, 78 insertions(+), 50 deletions(-) diff --git a/zfs_backup.sh b/zfs_backup.sh index c97191d..fdaf207 100755 --- a/zfs_backup.sh +++ b/zfs_backup.sh @@ -1,11 +1,5 @@ #!/bin/sh -if ! lockfile-create -p -r 2 $0; then - echo "Another backup is running, aborting" - exit 49 -fi - - ### Arguments ### DATASET=$1 if test -z "$DATASET"; then @@ -40,21 +34,33 @@ fi SNAPSHOT="$DATASET@$NAME" PREVIOUS_SNAPSHOT="$DATASET@$PREVIOUS_NAME" REMOTE_NAME="$TARGET_PATH/"$(echo -e $DATASET | sed -r "s/\//_/g") +LOCK_NAME="${0}_"$(echo -e $DATASET | sed -r "s/\//_/g") + +### LOCK ### +if ! lockfile-create -p -r 2 $LOCK_NAME; then + echo "Another backup is running, aborting" + exit 49 +fi -# Take snapshot -echo "Creating $SNAPSHOT snapshot..." -zfs snapshot $SNAPSHOT || exit 1 -echo "Snapshot created." +# Create snapshot +exists=$(zfs list -H -t snapshot -o name $SNAPSHOT) +if [ "$exists" = "$SNAPSHOT" ]; then + echo "Snapshot $SNAPSHOT already exists, skip create." +else + echo "Creating $SNAPSHOT snapshot..." + zfs snapshot $SNAPSHOT || exit 1 + echo "Snapshot created." +fi # Send snapshot echo "Sending snapshot ($PREVIOUS_SNAPSHOT --- $SNAPSHOT) to $REMOTE_NAME ..." if test ! -z "$PREVIOUS_NAME"; then incr="-I $PREVIOUS_SNAPSHOT" fi -zfs send -v -p $incr $SNAPSHOT | ssh $TARGET zfs receive -F $REMOTE_NAME || exit 1 +zfs send -v -p $incr $SNAPSHOT | ssh $TARGET zfs receive -F -s $REMOTE_NAME || exit 1 echo "Snapshot sent." -# Remove lock -echo "Backup finished. Removing lock..." -lockfile-remove $0 +### UNLOCK ### +echo "Backup of $DATASET dataset finished. Removing lock..." +lockfile-remove $LOCK_NAME diff --git a/zfs_backup_dataset.sh b/zfs_backup_dataset.sh index e178cec..aee921d 100755 --- a/zfs_backup_dataset.sh +++ b/zfs_backup_dataset.sh @@ -18,52 +18,74 @@ if test -z "$TARGET_PATH"; then exit 1 fi -### New snapshot ### -NAME=$(date +%Y-%m-%d_%H-%M-%S) -SNAPSHOT="$DATASET@$NAME" +DATE=$4 +if test -z "$DATE"; then + DATE=$(date +%Y-%m-%d_%H-%M) +fi +REMOTE_NAME="$TARGET_PATH/"$(echo -e $DATASET | sed -r "s/\//_/g") + + +### Resume existing transfer ### +RESUME_TOKEN=$(ssh $TARGET zfs get -H -p receive_resume_token -o value "$REMOTE_NAME") +if [ "$RESUME_TOKEN" != "" ] && [ "$RESUME_TOKEN" != "-" ]; then + echo "Resuming interrupted transfer..." + zfs send -v -t "$RESUME_TOKEN" | ssh $TARGET zfs receive -F -s "$REMOTE_NAME" || exit 1 + echo "Resumed transfer finished." + exit 0 +fi + + +### Sync existing snapshots ### +REMOTE_SNAPSHOTS=($(ssh $TARGET zfs list -H -t snapshot -o name -s creation $REMOTE_NAME | cut -d"@" -f 2)) +LOCAL_SNAPSHOTS=($(zfs list -H -t snapshot -o name -s creation $DATASET | cut -d"@" -f 2)) + +echo "Remote snapshots: $REMOTE_SNAPSHOTS" + +for local_snapshot in "${LOCAL_SNAPSHOTS[@]}"; do + echo "Syncing local snapshot $local_snapshot ..." + found=false + for remote_snapshot in "${REMOTE_SNAPSHOTS[@]}"; do + if [ "$remote_snapshot" = "$local_snapshot" ] ; then + found=true + break + fi + done + + if [ $found = true ] ; then + echo "> Found remotely, settings as last snapshot" + LAST_SNAPSHOT=$local_snapshot + EXISTING_SNAPSHOT="" + else + echo "> Needs sync" + EXISTING_SNAPSHOT=$local_snapshot + fi +done + + +### New snapshot ### +if [ "$EXISTING_SNAPSHOT" = "" ]; then + NAME=$DATE +else + NAME=$EXISTING_SNAPSHOT +fi +SNAPSHOT="$DATASET@$NAME" echo "New snapshot name is: $NAME" echo "New snapshot is: $SNAPSHOT" -### Default last snapshot ### -DEFAULT_LAST_SNAPSHOT=$(zfs list -t snapshot -o name -s creation -H $DATASET | tail -n 1) -DEFAULT_LAST_NAME=$(echo -e $DEFAULT_LAST_SNAPSHOT | rev | cut -d"@" -f1 | rev) -echo "Default last snapshot is: $DEFAULT_LAST_SNAPSHOT" -echo "Default last snapshot name is: $DEFAULT_LAST_NAME" - - -### Tracking folder ### -tracking_folder="$HOME/.local/share/zfs_backup" -if test ! -d $tracking_folder; then - echo "Creating backup tracking files directory" - mkdir -p $tracking_folder -fi - - ### Last snapshot ### -LAST_SNAPSHOT=$DEFAULT_LAST_SNAPSHOT - -last_snapshot_file="$tracking_folder/last_snapshot_"$(echo -e $DATASET | sed -r "s/\//_/g") -if test -f $last_snapshot_file; then - LAST_SNAPSHOT=$(cat $last_snapshot_file) - echo "Found $last_snapshot_file" -else - echo "No $last_snapshot_file, resolving to default" -fi - -LAST_SNAPSHOT_NAME=$(echo -e $LAST_SNAPSHOT | rev | cut -d"@" -f1 | rev) - echo "Last snapshot is: $LAST_SNAPSHOT" -echo "Last snapshot name is: $LAST_SNAPSHOT_NAME" -echo "Starting backup with args $DATASET $NAME $LAST_SNAPSHOT_NAME" -$HOME/scripts/zfs_backup.sh $DATASET $NAME $LAST_SNAPSHOT_NAME $TARGET $TARGET_PATH || exit 1 -echo "Saving new snapshot in tracking folder..." -echo -e $SNAPSHOT > $last_snapshot_file +## Perform transfer +echo "Starting backup with args $DATASET $NAME $LAST_SNAPSHOT" +$HOME/scripts/zfs_backup.sh $DATASET $NAME "$LAST_SNAPSHOT" $TARGET $TARGET_PATH || exit 1 + +## Destroy last snapshot echo "Destroying last snapshot..." -zfs destroy $LAST_SNAPSHOT || exit 1 +zfs destroy "$DATASET@$LAST_SNAPSHOT" || exit 1 + echo "Backup finished."