From b97eed404ababa32f518044c2f05f17f7b9089f5 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Sat, 19 Oct 2019 18:34:14 +0200 Subject: [PATCH] wip --- zfs_autobackup | 216 ++++++++++++++++++++++++------------------------- 1 file changed, 108 insertions(+), 108 deletions(-) diff --git a/zfs_autobackup b/zfs_autobackup index a7d704d..2a73489 100755 --- a/zfs_autobackup +++ b/zfs_autobackup @@ -85,11 +85,24 @@ class cached_property(object): self.__doc__ = getattr(func, '__doc__') self.func = func + def __get__(self, obj, cls): if obj is None: return self - value = obj.__dict__[self.func.__name__] = self.func(obj) - return value + + propname=self.func.__name__ + + #store directly in dict so its cached from now on + # value = obj.__dict__[propname] = self.func(obj) + if not hasattr(obj, '_cached_properties'): + obj._cached_properties={} + + if not propname in obj._cached_properties: + obj._cached_properties[propname]=self.func(obj) + # value = obj.__dict__[propname] = self.func(obj) + + return obj._cached_properties[propname] + @@ -175,18 +188,32 @@ class ZfsDataset(): def __repr__(self): return("{}: {}".format(self.zfs_node, self.name)) + def __str__(self): + return(self.name) + + # def invalidate(self): + # """invalidate cached data""" + # for d in self.__dict__: + # if (self.__getattribute__) + + + def invalidate(self): + """clear cache""" + #TODO: nicer + self._cached_properties={} + + @property def filesystem_name(self): """filesystem part of the name""" - (filesystem, snapshot_name)=self.name.split("@") - return(filesystem) + (filesystem, snapshot_name)=self.name.split("@") + return(filesystem) @property def snapshot_name(self): """snapshot part of the name""" - (filesystem, snapshot_name)=self.name.split("@") - return(snapshot_name) - + (filesystem, snapshot_name)=self.name.split("@") + return(snapshot_name) @cached_property def properties(self): @@ -196,22 +223,22 @@ class ZfsDataset(): "zfs", "get", "all", "-H", "-o", "property,value", self.name ] - return(dict(self.zfs_node.run(tab_split=True, cmd=cmd, valid_exitcodes=[ 0 ]))) + return(dict(self.zfs_node.run(tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[ 0 ]))) - def is_unchanged(self): - """dataset is unchanged since latest snapshot?""" + def is_changed(self): + """dataset is changed since ANY latest snapshot ?""" if self.properties['written']=="0B" or self.properties.written['written']=="0": - return(True) - else: return(False) + else: + return(True) def is_ours(self): """return true if this snapshot is created by this backup_nanme""" - if re.match("^"+self.zfs_node.backup_name+"-[0-9]*$", self.snapshot_name): - return(True) - else: - return(False) + if re.match("^"+self.zfs_node.backup_name+"-[0-9]*$", self.snapshot_name): + return(True) + else: + return(False) def from_names(self, names): """convert a list of names to a list ZfsDatasets for this zfs_node""" @@ -221,28 +248,50 @@ class ZfsDataset(): return(ret) - @cached_property def snapshots(self): """get all snaphots of this dataset""" cmd=[ - "zfs", "list", "-d", "1", "-r", "-t" ,"snapshot", "-H", "-o", "name" + "zfs", "list", "-d", "1", "-r", "-t" ,"snapshot", "-H", "-o", "name", self.name ] - names=self.zfs_node.run(cmd=cmd) + names=self.zfs_node.run(cmd=cmd, readonly=True) return(self.from_names(names)) + @cached_property + def our_snapshots(self): + """get list of snapshots creates by us of this dataset""" + ret=[] + for snapshot in self.snapshots: + if snapshot.is_ours(): + ret.append(snapshot) + + return(ret) @cached_property - def recursive_datasets(path, types="filesystem,volume"): - """get all datasets recursively under us""" + def is_changed_ours(self): + """dataset is changed since OUR latest snapshot?""" - names=self.zfs_node.run(tab_split=False, valid_exitcodes=[ 0 ], cmd=[ + latest_snapshot=self.snapshots[:-1] + cmd=[ "zfs", "get","-H" ,"-ovalue", "written@"+latest_snapshot, self.name ] + output=self.zfs_node.run(tab_split=False, cmd=cmd, valid_exitcodes=[ 0 ]) + + if output[0]=="0B" or output[0]=="0": + return(False) + + return(True) + + @cached_property + def recursive_datasets(self, types="filesystem,volume"): + """get all datasets recursively under us""" + + names=self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[ 0 ], cmd=[ "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name ]) - return(self.from_names(names)) + return(self.from_names(names[1:])) + class ZfsNode(ExecuteNode): """a node that contains zfs datasets. implements global lowlevel zfs commands""" @@ -251,6 +300,29 @@ class ZfsNode(ExecuteNode): self.backup_name=backup_name ExecuteNode.__init__(self, ssh_to=ssh_to, readonly=readonly) + def new_snapshotname(self): + """determine uniq new snapshotname""" + return(self.backup_name+"-"+time.strftime("%Y%m%d%H%M%S")) + + + def consistent_snapshot(self, datasets, snapshot_name, allow_empty=True): + """create a consistent (atomic) snapshot of specified datasets. + + allow_empty: Allow empty snapshots. (compared to our latest snapshot) + """ + + cmd=[ "zfs", "snapshot" ] + + noop=True + for dataset in datasets: + if allow_empty or dataset.is_changed_ours(): + cmd.append(str(dataset)+"@"+snapshot_name) + dataset.invalidate() + noop=False + + if not noop: + self.run(cmd, readonly=False) + @cached_property def selected_datasets(self): @@ -294,78 +366,6 @@ class ZfsNode(ExecuteNode): -# -# class ZfsPool(TreeNode): -# """a zfs pool""" -# def __init__(self, *kwargs, **args): -# super().__init(*args, **kwargs) -# -# """determine filesystems that should be backupped by looking at the special autobackup-property""" -# def zfs_get_selected_filesystems(ssh_to, backup_name): -# #get all source filesystems that have the backup property -# source_filesystems=run(ssh_to=ssh_to, tab_split=True, cmd=[ -# "zfs", "get", "-t", "volume,filesystem", "-o", "name,value,source", "-s", "local,inherited", "-H", "autobackup:"+backup_name -# ]) -# -# #determine filesystems that should be actually backupped -# selected_filesystems=[] -# direct_filesystems=[] -# for source_filesystem in source_filesystems: -# (name,value,source)=source_filesystem -# if value=="false": -# verbose("* Ignored : {0} (disabled)".format(name)) -# -# else: -# if source=="local" and ( value=="true" or value=="child"): -# direct_filesystems.append(name) -# -# if source=="local" and value=="true": -# selected_filesystems.append(name) -# verbose("* Selected: {0} (direct selection)".format(name)) -# elif source.find("inherited from ")==0 and (value=="true" or value=="child"): -# inherited_from=re.sub("^inherited from ", "", source) -# if inherited_from in direct_filesystems: -# selected_filesystems.append(name) -# verbose("* Selected: {0} (inherited selection)".format(name)) -# else: -# verbose("* Ignored : {0} (already a backup)".format(name)) -# else: -# verbose("* Ignored : {0} (only childs)".format(name)) -# -# return(selected_filesystems) -# -# -# -# -# class ZfsSnapshot(Dataset): -# """A zfs snapshot""" -# def __init__(previous_snapshot=false, next_snapshot=fase, keep_time=false, timestamp=false, **kwargs, *args): -# super.__init__(**kargs, *args) -# self.timestamp -# self.keep_time -# self.previous_snapshot -# self.next_snapshot - - -# class ZfsBackupSource(): -# """backup source. -# -# contains high level backup source functions. -# -# these work with ZfsDataset and ZfsSnapshot objects. -# -# """ -# -# def __init__(self): -# self.node=ZfsNode(ssh_to=args.ssh_to) -# self.datasets={} -# self.snapshots={} -# -# -# def refresh(): -# """refresh all data by calling various zfs commands""" -# selected_filesystems=self.node.zfs_get_selected_filesystems() - ################################################################## ENTRY POINT @@ -374,8 +374,8 @@ import argparse parser = argparse.ArgumentParser( description='ZFS autobackup v2.4', epilog='When a filesystem fails, zfs_backup will continue and report the number of failures at that end. Also the exit code will indicate the number of failures.') -parser.add_argument('--ssh-source', default="local", help='Source host to get backup from. (user@hostname) Default %(default)s.') -parser.add_argument('--ssh-target', default="local", help='Target host to push backup to. (user@hostname) Default %(default)s.') +parser.add_argument('--ssh-source', default=None, help='Source host to get backup from. (user@hostname) Default %(default)s.') +parser.add_argument('--ssh-target', default=None, help='Target host to push backup to. (user@hostname) Default %(default)s.') parser.add_argument('--keep-source', type=int, default=30, help='Number of days to keep old snapshots on source. Default %(default)s.') parser.add_argument('--keep-target', type=int, default=30, help='Number of days to keep old snapshots on target. Default %(default)s.') parser.add_argument('backup_name', help='Name of the backup (you should set the zfs property "autobackup:backup-name" to true on filesystems you want to backup') @@ -407,20 +407,20 @@ parser.add_argument('--debug', action='store_true', help='debug output (shows co args = parser.parse_args() -node=ZfsNode(args.backup_name, ssh_to=args.ssh_source) +source_node=ZfsNode(args.backup_name, ssh_to=args.ssh_source, readonly=args.test) +target_node=ZfsNode(args.backup_name, ssh_to=args.ssh_target, readonly=args.test) -source_datasets=node.selected_datasets +source_datasets=source_node.selected_datasets + if not source_datasets: abort("No source filesystems selected, please do a 'zfs set autobackup:{0}=true' on {1}".format(args.backup_name,args.ssh_source)) +source_node.consistent_snapshot(source_datasets, source_node.new_snapshotname(), allow_empty=args.allow_empty) - -pprint.pprint(source_datasets) -print() -pprint.pprint(source_datasets[0].__dict__) - -print(source_datasets[0].properties['mountpoint']) -print(source_datasets[1].properties['mountpoint']) -print(source_datasets[0].properties['mountpoint']) -print(source_datasets[1].properties['mountpoint']) +# for source_dataset in source_datasets: +# print(source_dataset) +# print(source_dataset.recursive_datasets) +# +# +# pprint.pprint(ZfsDataset(node, "rpool").recursive_datasets)