fixing quota issues

README.md

@@ -23,6 +23,7 @@ It has the following features:
 * Easy to debug and has a test-mode. Actual unix commands are printed.
 * Keeps latest X snapshots remote and locally. (default 30, configurable)
 * Uses zfs-holds on important snapshots so they cant be accidentally destroyed.
+* Tries to work around quota issues by temporary clearing those properties during backup.
 * Easy installation:
   * Only one host needs the zfs_autobackup script. The other host just needs ssh and the zfs command.
   * Written in python and uses zfs-commands, no 3rd party dependency's or libraries.

zfs_autobackup

@@ -250,15 +250,23 @@ def zfs_release_snapshot(ssh_to, snapshot, tag=None):
 
 
 
-"""gets all properties of a filesystem"""
+"""gets all properties of a filesystem, as a dict"""
 def zfs_get_properties(ssh_to, filesystem):
     cmd=[
-        "zfs", "get", "all", "-H", "-o", "property,value", snapshot
+        "zfs", "get", "-H", "-o", "property,value", "all", filesystem
     ]
 
-    run(ssh_to=ssh_to, tab_split=False, cmd=cmd, valid_exitcodes=[ 0, 1 ])
+    return(dict(run(ssh_to=ssh_to, tab_split=True, cmd=cmd)))
 
 
+"""set zfs property"""
+def zfs_set_property(ssh_to, filesystem, property, value):
+    cmd=[
+        "zfs", "set", property+"="+value, filesystem
+    ]
+
+    return(run(ssh_to=ssh_to, test=args.test, cmd=cmd))
+
 
 """transfer a zfs snapshot from source to target. both can be either local or via ssh.
 
@@ -343,10 +351,12 @@ def zfs_transfer(ssh_source, source_filesystem, first_snapshot, second_snapshot,
     target_cmd.extend(["zfs", "recv", "-u" ])
 
     # filter certain properties on receive (usefull for linux->freebsd in some cases)
+    # (-x is not supported on all platforms)
     if args.filter_properties:
         for filter_property in args.filter_properties:
             target_cmd.extend([ "-x" , filter_property ])
 
+
     if args.debug:
         target_cmd.append("-v")
 
@@ -666,12 +676,30 @@ def zfs_autobackup():
                 #now actually send the snapshots
                 if not args.no_send:
 
-                    if send_snapshots and args.rollback and latest_target_snapshot:
+                    ### prepare to send
-                        #roll back any changes on target
+                    source_properties=zfs_get_properties(ssh_to=args.ssh_source, filesystem=source_filesystem)
-                        debug("Rolling back target to latest snapshot.")
+                    if latest_target_snapshot:
-                        run(ssh_to=args.ssh_target, test=args.test, cmd=["zfs", "rollback", target_filesystem+"@"+latest_target_snapshot ])
+                        target_properties=zfs_get_properties(ssh_to=args.ssh_target, filesystem=target_filesystem)
+                    else:
+                        #new filesystem, no target props yet
+                        target_properties={}
+
+                    # we have acutally something to send?
+                    if send_snapshots:
+                        #clear target quotas to prevent space issues during transfer.
+                        #these will be restored automaticly at the end.
+                        for property in ['quota', 'refquota' ]:
+                            if property in target_properties and target_properties[property]!='none':
+                                zfs_set_property(args.ssh_target, target_filesystem, property, 'none')
+
+                        #rollback?
+                        if args.rollback and latest_target_snapshot:
+                            #roll back any changes on target
+                            debug("Rolling back target to latest snapshot.")
+                            run(ssh_to=args.ssh_target, test=args.test, cmd=["zfs", "rollback", target_filesystem+"@"+latest_target_snapshot ])
 
 
+                    ### traverse all the snapshots and send them
                     for send_snapshot in send_snapshots:
 
                         #resumable?
@@ -684,6 +712,7 @@ def zfs_autobackup():
                         if not args.no_holds:
                             zfs_hold_snapshot(ssh_to=args.ssh_source, snapshot=source_filesystem+"@"+send_snapshot)
 
+                        #do the actual transfer
                         zfs_transfer(
                             ssh_source=args.ssh_source, source_filesystem=source_filesystem,
                             first_snapshot=latest_target_snapshot, second_snapshot=send_snapshot,
@@ -704,23 +733,44 @@ def zfs_autobackup():
                             if not args.no_holds:
                                 zfs_release_snapshot(ssh_to=args.ssh_source, snapshot=source_filesystem+"@"+latest_target_snapshot)
                                 source_obsolete_snapshots[source_filesystem].append(latest_target_snapshot)
-                        #we just received a new filesytem?
-                        else:
-                            if args.clear_refreservation:
-                                debug("Clearing refreservation to save space.")
-
-                                run(ssh_to=args.ssh_target, test=args.test, cmd=["zfs", "set", "refreservation=none", target_filesystem ])
-
-
-                            if args.clear_mountpoint:
-                                debug("Setting canmount=noauto to prevent auto-mounting in the wrong place. (ignoring errors)")
-
-                                run(ssh_to=args.ssh_target, test=args.test, cmd=["zfs", "set", "canmount=noauto", target_filesystem ], valid_exitcodes= [0, 1] )
-
 
                         latest_target_snapshot=send_snapshot
+
+                    ### finishedup sending, determine which target properties we need to set or copy
+                    if send_snapshots:
+                        #reread properties if we actually changed something
+                        target_properties=zfs_get_properties(ssh_to=args.ssh_target, filesystem=target_filesystem)
+
+                    new_target_properties={}
+                    if 'quota' in source_properties:
+                        new_target_properties['quota']=source_properties['quota']
+                    if 'refquota' in source_properties:
+                        new_target_properties['refquota']=source_properties['refquota']
+
+                    if 'refreservation' in source_properties:
+                        if args.clear_refreservation:
+                            new_target_properties['refreservation']='none'
+                        else:
+                            new_target_properties['refreservation']=source_properties['refreservation']
+
+                    if 'canmount' in source_properties:
+                        if args.clear_mountpoint:
+                            new_target_properties['canmount']='noauto'
+                        else:
+                            new_target_properties['canmount']=source_properties['canmount']
+
+                    #now set the target properties that are different
+                    for (property,value) in new_target_properties.items():
+                        if target_properties[property]!=value:
+                            verbose("Setting property on {}: {}={}".format(target_filesystem, property, value))
+                            zfs_set_property(args.ssh_target, target_filesystem, property, value)
+
+
+
         # failed, skip this source_filesystem
         except Exception as e:
+            if args.debug:
+                raise
             failed(str(e))
 
 
@@ -780,7 +830,7 @@ def zfs_autobackup():
 # parse arguments
 import argparse
 parser = argparse.ArgumentParser(
-    description='ZFS autobackup v2.4',
+    description='ZFS autobackup v2.5',
     epilog='When a filesystem fails, zfs_backup will continue and report the number of failures at that end. Also the exit code will indicate the number of failures.')
 parser.add_argument('--ssh-source', default="local", help='Source host to get backup from. (user@hostname) Default %(default)s.')
 parser.add_argument('--ssh-target', default="local", help='Target host to push backup to. (user@hostname) Default  %(default)s.')
@@ -789,7 +839,7 @@ parser.add_argument('--keep-target', type=int, default=30, help='Number of days
 parser.add_argument('backup_name',    help='Name of the backup (you should set the zfs property "autobackup:backup-name" to true on filesystems you want to backup')
 parser.add_argument('target_path',    help='Target ZFS filesystem')
 
-parser.add_argument('--no-snapshot', action='store_true', help='dont create new snapshot (usefull for finishing uncompleted backups, or cleanups)')
+parser.add_argument('--no-snapshot', action='store_true', help='Dont create new snapshot. Usefull for completing unfinished backups or to investigate a problem.')
 parser.add_argument('--no-send', action='store_true', help='dont send snapshots (usefull to only do a cleanup)')
 parser.add_argument('--allow-empty', action='store_true', help='if nothing has changed, still create empty snapshots.')
 parser.add_argument('--ignore-replicated', action='store_true',  help='Ignore datasets that seem to be replicated some other way. (No changes since lastest snapshot. Usefull for proxmox HA replication)')
@@ -811,7 +861,7 @@ parser.add_argument('--ignore-transfer-errors', action='store_true', help='Ignor
 
 parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)')
 parser.add_argument('--verbose', action='store_true', help='verbose output')
-parser.add_argument('--debug', action='store_true', help='debug output (shows commands that are executed)')
+parser.add_argument('--debug', action='store_true', help='debug output (shows commands that are executed, and aborts with a backtrace on the first error)')
 
 #note args is the only global variable we use, since its a global readonly setting anyway
 args = parser.parse_args()