ashpie/server-scripts
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zsc-sync.sh: reduce ssh calls by preloading remote info

Browse Source
This commit is contained in:
Alice Gaudon 2021-03-08 12:56:08 +01:00
parent b8f3cb9d1a
commit 18a7200bee
1 changed files with 18 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
zsc-sync.sh
View File

@ -26,12 +26,22 @@ if ! lockfile-create -p -r 2 "$0"; then
fi fi
echo "Fetching remote snapshots..."
remote_snapshots=$(ssh $REMOTE zfs list -H -t snapshot -o name -s creation)
echo "Fetching subdatasets..."
remote_subdatasets=$(ssh $REMOTE zfs list -H -o name -s creation -r "$REMOTE_ROOT_DATASET")
echo "Fetching resume tokens..."
resume_tokens=$(ssh $REMOTE zfs get -H -p receive_resume_token -o name,value -t filesystem)
### Resume existing transfer ### ### Resume existing transfer ###
# Returns 0 if no action was performed, 1 if a transfer was resumed and finished successfully, -1 if there was an error during resumed transfer # Returns 0 if no action was performed, 1 if a transfer was resumed and finished successfully, -1 if there was an error during resumed transfer
resume_transfer() { resume_transfer() {
remote_dataset=$1 remote_dataset=$1
RESUME_TOKEN=$(ssh $REMOTE zfs get -H -p receive_resume_token -o value "$remote_dataset") RESUME_TOKEN=$(echo -e "$resume_tokens" | grep -P "^$remote_dataset\t.*\$" | cut -d$'\t' -f2)
if [ "$RESUME_TOKEN" != "" ] && [ "$RESUME_TOKEN" != "-" ]; then if [ "$RESUME_TOKEN" != "" ] && [ "$RESUME_TOKEN" != "-" ]; then
echo "Resuming interrupted transfer..." echo "Resuming interrupted transfer..."
zfs send -v -t "$RESUME_TOKEN" | ssh $REMOTE zfs receive -F -s "$remote_dataset" || return -1 zfs send -v -t "$RESUME_TOKEN" | ssh $REMOTE zfs receive -F -s "$remote_dataset" || return -1
@ -60,7 +70,7 @@ handle_dataset() {
PREFIX="" PREFIX=""
fi fi
TARGET_REMOTE_DATASET="$REMOTE_ROOT_DATASET/"$(echo -e $DATASET | sed -r "s/\//_/g") TARGET_REMOTE_DATASET="$REMOTE_ROOT_DATASET/"$(echo -e "$DATASET" | sed -r "s/\//_/g")
echo "[$DATASET -> $TARGET_REMOTE_DATASET]" echo "[$DATASET -> $TARGET_REMOTE_DATASET]"
echo -e echo -e
@ -70,15 +80,14 @@ handle_dataset() {
### Sync existing snapshots ### ### Sync existing snapshots ###
REMOTE_SNAPSHOTS=($(ssh $REMOTE zfs list -H -t snapshot -o name -s creation $TARGET_REMOTE_DATASET | cut -d"@" -f 2)) REMOTE_SNAPSHOTS=($(echo -e "$remote_snapshots" | grep "$TARGET_REMOTE_DATASET@" | cut -d"@" -f 2))
LOCAL_SNAPSHOTS=($(zfs list -H -t snapshot -o name -s creation $DATASET | cut -d"@" -f 2)) LOCAL_SNAPSHOTS=($(zfs list -H -t snapshot -o name -s creation "$DATASET" | cut -d"@" -f 2))
echo "Remote snapshots: ${#REMOTE_SNAPSHOTS[@]}" echo "Remote snapshots: ${#REMOTE_SNAPSHOTS[@]}"
echo "Local snapshots: ${#LOCAL_SNAPSHOTS[@]}" echo "Local snapshots: ${#LOCAL_SNAPSHOTS[@]}"
echo -e echo -e
for local_snapshot in "${LOCAL_SNAPSHOTS[@]}"; do for local_snapshot in "${LOCAL_SNAPSHOTS[@]}"; do
echo "Syncing local snapshot $local_snapshot ..."
found=false found=false
for remote_snapshot in "${REMOTE_SNAPSHOTS[@]}"; do for remote_snapshot in "${REMOTE_SNAPSHOTS[@]}"; do
if [ "$remote_snapshot" = "$local_snapshot" ] ; then if [ "$remote_snapshot" = "$local_snapshot" ] ; then
@ -88,18 +97,16 @@ handle_dataset() {
done done
if [ $found = true ] ; then if [ $found = true ] ; then
echo "> Found remotely, setting as last snapshot" echo "> $local_snapshot found remotely"
FROM_SNAPSHOT=$local_snapshot FROM_SNAPSHOT=$local_snapshot
TO_SNAPSHOT="" TO_SNAPSHOT=""
else else
echo "> Needs sync" echo "> $local_snapshot needs sync"
TO_SNAPSHOT=$local_snapshot TO_SNAPSHOT=$local_snapshot
fi fi
echo -e
done done
echo "Snapshot are synchronized up to $FROM_SNAPSHOT" echo "Snapshots are synchronized up to $FROM_SNAPSHOT"
echo -e echo -e
if test ! -z "$TO_SNAPSHOT"; then if test ! -z "$TO_SNAPSHOT"; then
@ -122,21 +129,11 @@ init_subdataset() {
PARENT_DATASET=$2 PARENT_DATASET=$2
TARGET_REMOTE_DATASET="$REMOTE_ROOT_DATASET/"$(echo -e $PARENT_DATASET | sed -r "s/\//_/g")"/"${DATASET:$(echo -e "$PARENT_DATASET" | wc -c)} TARGET_REMOTE_DATASET="$REMOTE_ROOT_DATASET/"$(echo -e $PARENT_DATASET | sed -r "s/\//_/g")"/"${DATASET:$(echo -e "$PARENT_DATASET" | wc -c)}
REMOTE_DATASETS=($(ssh $REMOTE zfs list -H -o name -s creation -r $REMOTE_ROOT_DATASET))
resume_transfer "$TARGET_REMOTE_DATASET" || return 0 resume_transfer "$TARGET_REMOTE_DATASET" || return 0
if [ $(echo -e "$remote_subdatasets" | grep -e "^$TARGET_REMOTE_DATASET\$" | wc -l) -eq 0 ]; then
found=false
for d in "${REMOTE_DATASETS[@]}"; do
if [ "$TARGET_REMOTE_DATASET" == "$d" ]; then
found=true
break
fi
done
if [ $found = false ]; then
echo "Subdataset not found on $TARGET_REMOTE_DATASET, initializing..." echo "Subdataset not found on $TARGET_REMOTE_DATASET, initializing..."
LOCAL_SNAPSHOT=($(zfs list -H -t snapshot -o name -s creation "$DATASET" | cut -d"@" -f 2 | tail -n 1)) LOCAL_SNAPSHOT=($(zfs list -H -t snapshot -o name -s creation "$DATASET" | cut -d"@" -f 2 | tail -n 1))