mirror of
https://github.com/psy0rz/zfs_autobackup.git
synced 2025-04-11 22:40:01 +03:00
now also deletes old snapshots from stale filesystems. working on perfecting --destroy-stale. (not 100% safe now, so check what will be destroyed before you use it)
This commit is contained in:
Edwin Eefting
parent
9c0def42ca
commit
51f8f08503
183
zfs_autobackup
183
zfs_autobackup
@ -11,26 +11,6 @@ import pprint
|
||||
import cStringIO
|
||||
import time
|
||||
|
||||
###### parse arguments
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='ZFS autobackup v2.0')
|
||||
parser.add_argument('--ssh-source', default="local", help='Source host to get backup from. (user@hostname) Default %(default)s.')
|
||||
parser.add_argument('--ssh-target', default="local", help='Target host to push backup to. (user@hostname) Default %(default)s.')
|
||||
parser.add_argument('--ssh-cipher', default="arcfour128", help='SSH cipher to use (default %(default)s)')
|
||||
parser.add_argument('--keep-source', type=int, default=30, help='Number of days to keep old snapshots on source. Default %(default)s.')
|
||||
parser.add_argument('--keep-target', type=int, default=30, help='Number of days to keep old snapshots on target. Default %(default)s.')
|
||||
parser.add_argument('backup_name', help='Name of the backup (you should set the zfs property "autobackup:backup-name" to true on filesystems you want to backup')
|
||||
parser.add_argument('target_fs', help='Target filesystem')
|
||||
|
||||
parser.add_argument('--no-snapshot', action='store_true', help='dont create new snapshot (usefull for finishing uncompleted backups, or cleanups)')
|
||||
parser.add_argument('--no-send', action='store_true', help='dont send snapshots (usefull to only do a cleanup)')
|
||||
|
||||
parser.add_argument('--compress', action='store_true', help='use compression during zfs send/recv')
|
||||
parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)')
|
||||
parser.add_argument('--verbose', action='store_true', help='verbose output')
|
||||
parser.add_argument('--debug', action='store_true', help='debug output (shows commands that are executed)')
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def error(txt):
|
||||
print(txt, file=sys.stderr)
|
||||
@ -132,30 +112,30 @@ def zfs_get_selected_filesystems(ssh_to, backup_name):
|
||||
|
||||
|
||||
|
||||
"""determine filesystems that where already backupped to the target. (used to determine cleanup list)"""
|
||||
def zfs_get_backupped_filesystems(ssh_to, backup_name, target_fs):
|
||||
#get all target filesystems that have the backup property and have a receive autobackup property
|
||||
target_filesystems=run(ssh_to=ssh_to, tab_split=True, cmd=[
|
||||
"zfs", "get", "-r", "-t", "volume,filesystem", "-o", "name,value,source", "-s", "received,inherited", "-H", "autobackup:"+backup_name, taget_fs
|
||||
])
|
||||
"""deferred destroy list of snapshots (in @format). """
|
||||
def zfs_destroy_snapshots(ssh_to, snapshots):
|
||||
|
||||
|
||||
|
||||
|
||||
"""destroy list of filesystems or snapshots (in @format) """
|
||||
def zfs_destroy(ssh_to, filesystems):
|
||||
|
||||
# debug("Destroying on {0}:\n{1}".format(ssh_to, "\n".join(filesystems)))
|
||||
#zfs can only destroy one filesystem at once so we use xargs and stdin
|
||||
run(ssh_to=ssh_to, test=args.test, input="\0".join(filesystems), cmd=
|
||||
run(ssh_to=ssh_to, test=args.test, input="\0".join(snapshots), cmd=
|
||||
[ "xargs", "-0", "-n", "1", "zfs", "destroy", "-d" ]
|
||||
)
|
||||
|
||||
"""destroy list of filesystems """
|
||||
def zfs_destroy(ssh_to, filesystems, recursive=False):
|
||||
|
||||
cmd=[ "xargs", "-0", "-n", "1", "zfs", "destroy" ]
|
||||
|
||||
if recursive:
|
||||
cmd.append("-r")
|
||||
|
||||
#zfs can only destroy one filesystem at once so we use xargs and stdin
|
||||
run(ssh_to=ssh_to, test=args.test, input="\0".join(filesystems), cmd=cmd)
|
||||
|
||||
#simulate snapshots for --test option
|
||||
test_snapshots={}
|
||||
|
||||
|
||||
|
||||
"""create snapshot on multiple filesystems at once (atomicly)"""
|
||||
def zfs_create_snapshot(ssh_to, filesystems, snapshot):
|
||||
cmd=[ "zfs", "snapshot" ]
|
||||
@ -300,8 +280,85 @@ def zfs_transfer(ssh_source, source_filesystem, first_snapshot, second_snapshot,
|
||||
|
||||
|
||||
|
||||
"""get filesystems that where already backupped to a target. """
|
||||
def zfs_get_backupped_filesystems(ssh_to, backup_name, target_fs):
|
||||
#get all target filesystems that have received or inherited the backup propert, under the target_fs tree
|
||||
ret=run(ssh_to=ssh_to, tab_split=False, cmd=[
|
||||
"zfs", "get", "-r", "-t", "volume,filesystem", "-o", "name", "-s", "received,inherited", "-H", "autobackup:"+backup_name, target_fs
|
||||
])
|
||||
|
||||
return(ret)
|
||||
|
||||
|
||||
|
||||
"""get filesystems that where once backupped to target but are no longer selected on source
|
||||
|
||||
these are filesystems that are not in the list in target_filesystems.
|
||||
|
||||
this happens when filesystems are destroyed or unselected on the source.
|
||||
"""
|
||||
def get_stale_backupped_filesystems(ssh_to, backup_name, target_fs, target_filesystems):
|
||||
|
||||
backupped_filesystems=zfs_get_backupped_filesystems(ssh_to=ssh_to, backup_name=backup_name, target_fs=target_fs)
|
||||
|
||||
#determine backupped filesystems that are not in target_filesystems anymore
|
||||
stale_backupped_filesystems=[]
|
||||
for backupped_filesystem in backupped_filesystems:
|
||||
if backupped_filesystem not in target_filesystems:
|
||||
stale_backupped_filesystems.append(backupped_filesystem)
|
||||
|
||||
return(stale_backupped_filesystems)
|
||||
|
||||
|
||||
now=time.time()
|
||||
"""determine list of snapshot (in @format) to destroy, according to age"""
|
||||
def determine_destroy_list(snapshots, days):
|
||||
ret=[]
|
||||
for filesystem in snapshots:
|
||||
for snapshot in snapshots[filesystem]:
|
||||
time_str=re.findall("^.*-([0-9]*)$", snapshot)[0]
|
||||
if len(time_str)==14:
|
||||
#new format:
|
||||
time_secs=time.mktime(time.strptime(time_str,"%Y%m%d%H%M%S"))
|
||||
else:
|
||||
time_secs=int(time_str)
|
||||
# verbose("time_secs"+time_str)
|
||||
if (now-time_secs) > (24 * 3600 * days):
|
||||
ret.append(filesystem+"@"+snapshot)
|
||||
|
||||
return(ret)
|
||||
|
||||
|
||||
################################################################## ENTRY POINT
|
||||
|
||||
|
||||
|
||||
############## parse arguments
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='ZFS autobackup v2.0')
|
||||
parser.add_argument('--ssh-source', default="local", help='Source host to get backup from. (user@hostname) Default %(default)s.')
|
||||
parser.add_argument('--ssh-target', default="local", help='Target host to push backup to. (user@hostname) Default %(default)s.')
|
||||
parser.add_argument('--ssh-cipher', default="arcfour128", help='SSH cipher to use (default %(default)s)')
|
||||
parser.add_argument('--keep-source', type=int, default=30, help='Number of days to keep old snapshots on source. Default %(default)s.')
|
||||
parser.add_argument('--keep-target', type=int, default=30, help='Number of days to keep old snapshots on target. Default %(default)s.')
|
||||
parser.add_argument('backup_name', help='Name of the backup (you should set the zfs property "autobackup:backup-name" to true on filesystems you want to backup')
|
||||
parser.add_argument('target_fs', help='Target filesystem')
|
||||
|
||||
parser.add_argument('--no-snapshot', action='store_true', help='dont create new snapshot (usefull for finishing uncompleted backups, or cleanups)')
|
||||
parser.add_argument('--no-send', action='store_true', help='dont send snapshots (usefull to only do a cleanup)')
|
||||
|
||||
parser.add_argument('--destroy-stale', action='store_true', help='Destroy stale backups that have no more snapshots. Be sure to verify the output before using this! ')
|
||||
|
||||
parser.add_argument('--compress', action='store_true', help='use compression during zfs send/recv')
|
||||
parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)')
|
||||
parser.add_argument('--verbose', action='store_true', help='verbose output')
|
||||
parser.add_argument('--debug', action='store_true', help='debug output (shows commands that are executed)')
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
|
||||
############## data gathering section
|
||||
|
||||
if args.test:
|
||||
args.verbose=True
|
||||
verbose("RUNNING IN TEST-MODE, NOT MAKING ACTUAL BACKUP!")
|
||||
@ -316,7 +373,7 @@ if not source_filesystems:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
#determine target filesystems
|
||||
#determine target filesystems (just append args.target_fs prefix)
|
||||
target_filesystems=[]
|
||||
for source_filesystem in source_filesystems:
|
||||
target_filesystems.append(args.target_fs+"/"+source_filesystem)
|
||||
@ -327,7 +384,7 @@ if not args.no_snapshot:
|
||||
verbose("Creating source snapshot {0} on {1} ".format(new_snapshot_name, args.ssh_source))
|
||||
zfs_create_snapshot(args.ssh_source, source_filesystems, new_snapshot_name)
|
||||
|
||||
#determine all snapshots of all selected filesystems on both source and target
|
||||
#get all snapshots of all selected filesystems on both source and target
|
||||
verbose("Getting source snapshot-list from {0}".format(args.ssh_source))
|
||||
source_snapshots=zfs_get_snapshots(args.ssh_source, source_filesystems, args.backup_name)
|
||||
debug("Source snapshots: " + str(pprint.pformat(source_snapshots)))
|
||||
@ -346,6 +403,10 @@ debug("Target snapshots: " + str(pprint.pformat(target_snapshots)))
|
||||
source_obsolete_snapshots={}
|
||||
target_obsolete_snapshots={}
|
||||
|
||||
|
||||
|
||||
############## backup section
|
||||
|
||||
#determine which snapshots to send for each filesystem
|
||||
for source_filesystem in source_filesystems:
|
||||
target_filesystem=args.target_fs+"/"+source_filesystem
|
||||
@ -395,36 +456,46 @@ for source_filesystem in source_filesystems:
|
||||
|
||||
latest_target_snapshot=send_snapshot
|
||||
|
||||
|
||||
|
||||
############## cleanup section
|
||||
#we only do cleanups after everything is complete, to keep everything consistent (same snapshots everywhere)
|
||||
|
||||
#get list of snapshot (in @format) to destroy
|
||||
now=time.time()
|
||||
def get_destroy_list(snapshots, days):
|
||||
ret=[]
|
||||
for filesystem in snapshots:
|
||||
for snapshot in snapshots[filesystem]:
|
||||
time_str=re.findall("^.*-([0-9]*)$", snapshot)[0]
|
||||
if len(time_str)==14:
|
||||
#new format:
|
||||
time_secs=time.mktime(time.strptime(time_str,"%Y%m%d%H%M%S"))
|
||||
else:
|
||||
time_secs=int(time_str)
|
||||
# verbose("time_secs"+time_str)
|
||||
if (now-time_secs) > (24 * 3600 * days):
|
||||
ret.append(filesystem+"@"+snapshot)
|
||||
|
||||
return(ret)
|
||||
#find stale backups on target that have become obsolete
|
||||
verbose("Getting stale filesystems and snapshots from {0}".format(args.ssh_target))
|
||||
stale_target_filesystems=get_stale_backupped_filesystems(ssh_to=args.ssh_target, backup_name=args.backup_name, target_fs=args.target_fs, target_filesystems=target_filesystems)
|
||||
debug("Stale target filesystems: {0}".format("\n".join(stale_target_filesystems)))
|
||||
|
||||
stale_target_snapshots=zfs_get_snapshots(args.ssh_target, stale_target_filesystems, args.backup_name)
|
||||
debug("Stale target snapshots: " + str(pprint.pformat(stale_target_snapshots)))
|
||||
target_obsolete_snapshots.update(stale_target_snapshots)
|
||||
|
||||
#determine stale filesystems that have no snapshots left (the can be destroyed)
|
||||
#TODO: prevent destroying filesystems that have underlying filesystems that are still active.
|
||||
stale_target_destroys=[]
|
||||
for stale_target_filesystem in stale_target_filesystems:
|
||||
if stale_target_filesystem not in stale_target_snapshots:
|
||||
stale_target_destroys.append(stale_target_filesystem)
|
||||
|
||||
if stale_target_destroys:
|
||||
if args.destroy_stale:
|
||||
verbose("Destroying stale filesystems on target {0}:\n{1}".format(args.ssh_target, "\n".join(stale_target_destroys)))
|
||||
zfs_destroy(ssh_to=args.ssh_target, filesystems=stale_target_destroys, recursive=True)
|
||||
else:
|
||||
verbose("Stale filesystems on {0}, use --destroy-stale to destroy:\n{1}".format(args.ssh_target, "\n".join(stale_target_destroys)))
|
||||
|
||||
|
||||
source_destroys=get_destroy_list(source_obsolete_snapshots, args.keep_source)
|
||||
#now actually destroy the old snapshots
|
||||
source_destroys=determine_destroy_list(source_obsolete_snapshots, args.keep_source)
|
||||
if source_destroys:
|
||||
verbose("Destroying old snapshots on source {0}:\n{1}".format(args.ssh_source, "\n".join(source_destroys)))
|
||||
zfs_destroy(ssh_to=args.ssh_source, filesystems=source_destroys)
|
||||
zfs_destroy_snapshots(ssh_to=args.ssh_source, snapshots=source_destroys)
|
||||
|
||||
target_destroys=get_destroy_list(target_obsolete_snapshots, args.keep_target)
|
||||
target_destroys=determine_destroy_list(target_obsolete_snapshots, args.keep_target)
|
||||
if target_destroys:
|
||||
verbose("Destroying old snapshots on target {0}:\n{1}".format(args.ssh_target, "\n".join(target_destroys)))
|
||||
zfs_destroy(ssh_to=args.ssh_target, filesystems=target_destroys)
|
||||
zfs_destroy_snapshots(ssh_to=args.ssh_target, snapshots=target_destroys)
|
||||
|
||||
|
||||
verbose("All done")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user