zfsbk.sh: WIP write to separate dataset files; upload to backblaze

zfsbk.sh

@@ -9,6 +9,13 @@ bkname=${1:-"default"}
 # set to 1 to avoid incrementals and always take full dumps.
 max_incremental=${2:-"4"}
 
+# names of the DATASETs to exclude (datasets, not mountpoints!)
+# can override this list at runtime with EXCLUDES envvar.
+# can extend this list at runtime with EXTRA_EXCLUDES envvar.
+excludes=${EXCLUDES:-"rpool rpool/ROOT rpool/data"}
+
+UPLOAD_PATH="b2://arclightbackup/zfssnap"
+
 usage () {
     echo "Usage:"
     echo "zfsbk.sh <bkname> [numincr]"
@@ -40,56 +47,114 @@ echo "$max_incremental" | grep -qE '^[1-9][0-9]*$' || failmsg "Invalid number of
 export PATH=$PATH:/usr/local/bin
 
 # list snapshots by decreasing time. Arguments: [name] restrict listing to this tag
-list_named_snaps () {
+# list_named_snaps () {
+#     local bkname=$1
+#     snapdir="/.zfs/snapshot"
+#     for i in $snapdir/zbk-$bkname-*
+#     do
+#         test -e "$i" && (echo $i |sed "s@^$snapdir/@@")
+#     done | sort -r
+# }
+
+# list all snapshots for the tag by decreasing time.
+# Arguments:
+#   [name] restrict listing to this tag
+#   [set] restrict listing to this set
+all_snaps () {
     local bkname=$1
-    snapdir="/.zfs/snapshot"
+    zfs list -t snapshot | grep $bkname | cut -f1 -d' ' | sort -r
-    for i in $snapdir/zbk-$bkname-*
+}
+
+# list snapshots for the set by decreasing time.
+# Arguments:
+#   [name] restrict listing to this tag
+#   [set] restrict listing to this set
+snaps_for_set () {
+    local bkname=$1
+    local set=$2
+    zfs list -t snapshot | grep $bkname | grep $set | cut -f1 -d' ' | sort -r
+}
+
+# list datasets with snapshots. Arguments: [name] restrict listing to this tag
+list_snap_sets () {
+    local bkname=$1
+    zfs list -t snapshot | grep $bkname | cut -f1 -d' ' | cut -f1 -d'@' | uniq
+}
+
+# list the datasets we want to backup.
+# this is snap sets minus excludes
+list_backup_snap_sets () {
+    local bkname=$1
+    list_snap_sets $bkname | while read set
     do
-        test -e "$i" && (echo $i |sed "s@^$snapdir/@@")
+        matched=0
-    done | sort -r
+        for excl in $excludes
+        do
+            if [ "x$set" = "x$excl" ]
+            then
+                matched=1
+                break
+            fi
+        done
+        if [ $matched -ne 1 ]
+        then
+            echo -n "$set "
+        fi
+    done
 }
 
 # remove all datasets for a given tag. Arguments: $1 -> bkname
 reset_sequence () {
     local bkname=$1
-    list_named_snaps $bkname | while read snapname
+    all_snaps $bkname | while read snapname
     do
-        zfs destroy -r $zpool@$snapname
+        zfs destroy $snapname
     done
 }
 
 # take snapshot
-#echo ./zfssnap.sh $bkname $max_incremental
+echo ./zfssnap.sh $bkname $max_incremental
 if ! ./zfssnap.sh $bkname $max_incremental
 then
     echo "Error creating snapshot! Code $?"
     exit 1
 fi
 
-# get label
+# iterate all the sets we want to back up
-num_snaps=`list_named_snaps $bkname | awk 'END {print NR}'`
+# and write dump files
-label_latest=`list_named_snaps $bkname | awk NR==1`
+num_snaps=0 # save for later
+for set in `list_backup_snap_sets $bkname` ; do
+    snaps=`snaps_for_set $bkname $set`
+    num_snaps=`echo $snaps | awk 'BEGIN {RS=" "} END {print NR}'`
+    label_latest=`echo $snaps | awk 'BEGIN {RS=" "} NR==1'`
+    #echo "$set: $num_snaps snaps, latest: $label_latest"
 
-# decide what to do based on how many snapshots were found
+    # decide what to do based on how many snapshots were found
-if [ $num_snaps -eq 0 ]
+    if [ $num_snaps -eq 0 ]
-then
+    then
-    # no snapshots. Something's wrong
+        # no snapshots. Something's wrong
-    echo "Error, no snapshots found for $bkname!"
+        echo "Error, no snapshots found for $set $bkname!"
-    list_named_snaps $bkname
+        echo $snaps
-    exit 1
+        exit 1
-elif [ $num_snaps -eq 1 ]
+    elif [ $num_snaps -eq 1 ]
-then
+    then
-    # first snapshot. Send full
+        # first snapshot. Send full
-    bkfile="/backups/${label_latest}.dump"
+        echo "first snapshot. Send full"
-    #echo zfs send -R $zpool@$label_latest
+        file_latest=`echo $label_latest | sed 's@/@_@g'`
-    zfs send -R $zpool@$label_latest 2>/dev/null > $bkfile
+        bkfile="/backups/${file_latest}.dump"
-else
+        echo "zfs send -R $label_latest > $bkfile"
-    # n-th snapshot. Send incrementally
+        zfs send -R $label_latest 2>/dev/null > $bkfile
-    label_2ndlatest=`list_named_snaps $bkname | awk NR==2`
+    else
-    bkfile="/backups/${label_latest}--${label_2ndlatest}.dump"
+        # n-th snapshot. Send incrementally
-    #echo zfs send -i $label_2ndlatest -R $zpool@$label_latest
+        echo "n-th snapshot. Send incrementally"
-    zfs send -i $label_2ndlatest -R $zpool@$label_latest 2>/dev/null > $bkfile
+        file_latest=`echo $label_latest | sed 's@/@_@g'`
-fi
+        label_2ndlatest=`echo $snaps | awk 'BEGIN {RS=" "} NR==2'`
+        file_2ndlatest=`echo $label_2ndlatest | sed 's@/@_@g'`
+        bkfile="/backups/${file_latest}--${file_2ndlatest}.dump"
+        echo "zfs send -i $label_2ndlatest -R $label_latest > $bkfile"
+        zfs send -i $label_2ndlatest -R $label_latest 2>/dev/null > $bkfile
+    fi
+done
 
 if [ $num_snaps -eq $max_incremental ]
 then
@@ -118,6 +183,13 @@ then
         fi
         scp -BCq $bkfile "$SCP_PATH"
         upload_excode=$?
+    elif echo "$UPLOAD_PATH" | grep -qE '^b2://'
+    then
+        #B2_PATH=${UPLOAD_PATH#b2://}
+        echo "Uploading to b2: $bkfile"
+        #echo "b2 upload-file --noProgress $bkfile zfssnap/$B2_PATH"
+        echo "b2 sync --delete --noProgress /backups $UPLOAD_PATH"
+        upload_excode=$?
     else
         echo "UPLOAD_PATH not understood! '$UPLOAD_PATH'"
         echo "Expecting rsync://.. or scp://.."