diff --git a/README.md b/README.md index f701d1d..efcab22 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ It has the following features: * Easy to debug and has a test-mode. Actual unix commands are printed. * Keeps latest X snapshots remote and locally. (default 30, configurable) * Uses zfs-holds on important snapshots so they cant be accidentally destroyed. +* Tries to work around quota issues by temporary clearing those properties during backup. * Easy installation: * Only one host needs the zfs_autobackup script. The other host just needs ssh and the zfs command. * Written in python and uses zfs-commands, no 3rd party dependency's or libraries. diff --git a/zfs_autobackup b/zfs_autobackup index e87ec21..dbfccfa 100755 --- a/zfs_autobackup +++ b/zfs_autobackup @@ -250,15 +250,23 @@ def zfs_release_snapshot(ssh_to, snapshot, tag=None): -"""gets all properties of a filesystem""" +"""gets all properties of a filesystem, as a dict""" def zfs_get_properties(ssh_to, filesystem): cmd=[ - "zfs", "get", "all", "-H", "-o", "property,value", snapshot + "zfs", "get", "-H", "-o", "property,value", "all", filesystem ] - run(ssh_to=ssh_to, tab_split=False, cmd=cmd, valid_exitcodes=[ 0, 1 ]) + return(dict(run(ssh_to=ssh_to, tab_split=True, cmd=cmd))) +"""set zfs property""" +def zfs_set_property(ssh_to, filesystem, property, value): + cmd=[ + "zfs", "set", property+"="+value, filesystem + ] + + return(run(ssh_to=ssh_to, test=args.test, cmd=cmd)) + """transfer a zfs snapshot from source to target. both can be either local or via ssh. @@ -343,10 +351,12 @@ def zfs_transfer(ssh_source, source_filesystem, first_snapshot, second_snapshot, target_cmd.extend(["zfs", "recv", "-u" ]) # filter certain properties on receive (usefull for linux->freebsd in some cases) + # (-x is not supported on all platforms) if args.filter_properties: for filter_property in args.filter_properties: target_cmd.extend([ "-x" , filter_property ]) + if args.debug: target_cmd.append("-v") @@ -666,12 +676,30 @@ def zfs_autobackup(): #now actually send the snapshots if not args.no_send: - if send_snapshots and args.rollback and latest_target_snapshot: - #roll back any changes on target - debug("Rolling back target to latest snapshot.") - run(ssh_to=args.ssh_target, test=args.test, cmd=["zfs", "rollback", target_filesystem+"@"+latest_target_snapshot ]) + ### prepare to send + source_properties=zfs_get_properties(ssh_to=args.ssh_source, filesystem=source_filesystem) + if latest_target_snapshot: + target_properties=zfs_get_properties(ssh_to=args.ssh_target, filesystem=target_filesystem) + else: + #new filesystem, no target props yet + target_properties={} + + # we have acutally something to send? + if send_snapshots: + #clear target quotas to prevent space issues during transfer. + #these will be restored automaticly at the end. + for property in ['quota', 'refquota' ]: + if property in target_properties and target_properties[property]!='none': + zfs_set_property(args.ssh_target, target_filesystem, property, 'none') + + #rollback? + if args.rollback and latest_target_snapshot: + #roll back any changes on target + debug("Rolling back target to latest snapshot.") + run(ssh_to=args.ssh_target, test=args.test, cmd=["zfs", "rollback", target_filesystem+"@"+latest_target_snapshot ]) + ### traverse all the snapshots and send them for send_snapshot in send_snapshots: #resumable? @@ -684,6 +712,7 @@ def zfs_autobackup(): if not args.no_holds: zfs_hold_snapshot(ssh_to=args.ssh_source, snapshot=source_filesystem+"@"+send_snapshot) + #do the actual transfer zfs_transfer( ssh_source=args.ssh_source, source_filesystem=source_filesystem, first_snapshot=latest_target_snapshot, second_snapshot=send_snapshot, @@ -704,23 +733,44 @@ def zfs_autobackup(): if not args.no_holds: zfs_release_snapshot(ssh_to=args.ssh_source, snapshot=source_filesystem+"@"+latest_target_snapshot) source_obsolete_snapshots[source_filesystem].append(latest_target_snapshot) - #we just received a new filesytem? - else: - if args.clear_refreservation: - debug("Clearing refreservation to save space.") - - run(ssh_to=args.ssh_target, test=args.test, cmd=["zfs", "set", "refreservation=none", target_filesystem ]) - - - if args.clear_mountpoint: - debug("Setting canmount=noauto to prevent auto-mounting in the wrong place. (ignoring errors)") - - run(ssh_to=args.ssh_target, test=args.test, cmd=["zfs", "set", "canmount=noauto", target_filesystem ], valid_exitcodes= [0, 1] ) - latest_target_snapshot=send_snapshot + + ### finishedup sending, determine which target properties we need to set or copy + if send_snapshots: + #reread properties if we actually changed something + target_properties=zfs_get_properties(ssh_to=args.ssh_target, filesystem=target_filesystem) + + new_target_properties={} + if 'quota' in source_properties: + new_target_properties['quota']=source_properties['quota'] + if 'refquota' in source_properties: + new_target_properties['refquota']=source_properties['refquota'] + + if 'refreservation' in source_properties: + if args.clear_refreservation: + new_target_properties['refreservation']='none' + else: + new_target_properties['refreservation']=source_properties['refreservation'] + + if 'canmount' in source_properties: + if args.clear_mountpoint: + new_target_properties['canmount']='noauto' + else: + new_target_properties['canmount']=source_properties['canmount'] + + #now set the target properties that are different + for (property,value) in new_target_properties.items(): + if target_properties[property]!=value: + verbose("Setting property on {}: {}={}".format(target_filesystem, property, value)) + zfs_set_property(args.ssh_target, target_filesystem, property, value) + + + # failed, skip this source_filesystem except Exception as e: + if args.debug: + raise failed(str(e)) @@ -780,7 +830,7 @@ def zfs_autobackup(): # parse arguments import argparse parser = argparse.ArgumentParser( - description='ZFS autobackup v2.4', + description='ZFS autobackup v2.5', epilog='When a filesystem fails, zfs_backup will continue and report the number of failures at that end. Also the exit code will indicate the number of failures.') parser.add_argument('--ssh-source', default="local", help='Source host to get backup from. (user@hostname) Default %(default)s.') parser.add_argument('--ssh-target', default="local", help='Target host to push backup to. (user@hostname) Default %(default)s.') @@ -789,7 +839,7 @@ parser.add_argument('--keep-target', type=int, default=30, help='Number of days parser.add_argument('backup_name', help='Name of the backup (you should set the zfs property "autobackup:backup-name" to true on filesystems you want to backup') parser.add_argument('target_path', help='Target ZFS filesystem') -parser.add_argument('--no-snapshot', action='store_true', help='dont create new snapshot (usefull for finishing uncompleted backups, or cleanups)') +parser.add_argument('--no-snapshot', action='store_true', help='Dont create new snapshot. Usefull for completing unfinished backups or to investigate a problem.') parser.add_argument('--no-send', action='store_true', help='dont send snapshots (usefull to only do a cleanup)') parser.add_argument('--allow-empty', action='store_true', help='if nothing has changed, still create empty snapshots.') parser.add_argument('--ignore-replicated', action='store_true', help='Ignore datasets that seem to be replicated some other way. (No changes since lastest snapshot. Usefull for proxmox HA replication)') @@ -811,7 +861,7 @@ parser.add_argument('--ignore-transfer-errors', action='store_true', help='Ignor parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)') parser.add_argument('--verbose', action='store_true', help='verbose output') -parser.add_argument('--debug', action='store_true', help='debug output (shows commands that are executed)') +parser.add_argument('--debug', action='store_true', help='debug output (shows commands that are executed, and aborts with a backtrace on the first error)') #note args is the only global variable we use, since its a global readonly setting anyway args = parser.parse_args()