From 6e8cb796fe9f246171bf9e47cf95a3163ff19157 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Sat, 21 Sep 2024 11:21:58 +0200 Subject: [PATCH 001/135] WIP: improve virtual snapshot code --- tests/test_zfsautobackup32.py | 2 +- zfs_autobackup/Thinner.py | 38 +++++---- zfs_autobackup/ZfsDataset.py | 142 ++++++++++++++++++++-------------- zfs_autobackup/ZfsNode.py | 30 ++++++- 4 files changed, 131 insertions(+), 81 deletions(-) diff --git a/tests/test_zfsautobackup32.py b/tests/test_zfsautobackup32.py index 75771ce..068d322 100644 --- a/tests/test_zfsautobackup32.py +++ b/tests/test_zfsautobackup32.py @@ -163,9 +163,9 @@ test_target1/test_source2/fs2/sub@test-20101111000001 with mocktime("20010202000000"): self.assertFalse(ZfsAutobackup("test2 --allow-empty".split(" ")).run()) - #will become common snapshot with OutputIO() as buf: with redirect_stdout(buf): + # now do thinning and transfer all at once with mocktime("20010203000000"): self.assertFalse(ZfsAutobackup("--keep-source=1d10d --keep-target=1m10m --allow-empty --verbose --clear-mountpoint --other-snapshots test2 test_target1".split(" ")).run()) diff --git a/zfs_autobackup/Thinner.py b/zfs_autobackup/Thinner.py index 0718505..1cbb3ea 100644 --- a/zfs_autobackup/Thinner.py +++ b/zfs_autobackup/Thinner.py @@ -70,24 +70,30 @@ class Thinner: # traverse objects for thisobject in objects: - # important they are ints! - timestamp = int(thisobject.timestamp) - age = int(now) - timestamp - # store in the correct time blocks, per period-size, if not too old yet - # e.g.: look if there is ANY timeblock that wants to keep this object - keep = False - for rule in self.rules: - if age <= rule.ttl: - block_nr = int(timestamp / rule.period) - if block_nr not in time_blocks[rule.period]: - time_blocks[rule.period][block_nr] = True - keep = True - - # keep it according to schedule, or keep it because it is in the keep_objects list - if keep or thisobject in keep_objects or thisobject in always_keep_objects: + #ignore stuff without timestamp, always keep those. + if thisobject.timestamp is None: keeps.append(thisobject) else: - removes.append(thisobject) + + # important they are ints! + timestamp = int(thisobject.timestamp) + age = int(now) - timestamp + + # store in the correct time blocks, per period-size, if not too old yet + # e.g.: look if there is ANY timeblock that wants to keep this object + keep = False + for rule in self.rules: + if age <= rule.ttl: + block_nr = int(timestamp / rule.period) + if block_nr not in time_blocks[rule.period]: + time_blocks[rule.period][block_nr] = True + keep = True + + # keep it according to schedule, or keep it because it is in the keep_objects list + if keep or thisobject in keep_objects or thisobject in always_keep_objects: + keeps.append(thisobject) + else: + removes.append(thisobject) return keeps, removes \ No newline at end of file diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index a8e83d4..7536a65 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -22,13 +22,12 @@ class ZfsDataset: def __init__(self, zfs_node, name, force_exists=None): """ Args: - :type zfs_node: ZfsNode.ZfsNode + :type zfs_node: ZfsNode :type name: str :type force_exists: bool """ self.zfs_node = zfs_node self.name = name # full name - self._virtual_snapshots = [] self.invalidate() self.force_exists = force_exists @@ -36,6 +35,7 @@ class ZfsDataset: return "{}: {}".format(self.zfs_node, self.name) def __str__(self): + return self.name def __eq__(self, obj): @@ -76,7 +76,6 @@ class ZfsDataset: """clear caches""" CachedProperty.clear(self) self.force_exists = None - self._virtual_snapshots = [] def split_path(self): """return the path elements as an array""" @@ -379,7 +378,7 @@ class ZfsDataset: return True def is_ours(self): - """return true if this snapshot name has format""" + """return true if this snapshot name belong to the current backup_name and snapshot formatting""" try: test = self.timestamp except ValueError as e: @@ -413,8 +412,14 @@ class ZfsDataset: @property def timestamp(self): """get timestamp from snapshot name. Only works for our own snapshots - with the correct format. + with the correct format. Snapshots that are not ours always return None + + :rtype: int|None """ + + if not self.is_ours(): + return None + dt = datetime.strptime(self.snapshot_name, self.zfs_node.snapshot_time_format) if sys.version_info[0] >= 3: from datetime import timezone @@ -432,17 +437,6 @@ class ZfsDataset: seconds = time.mktime(dt.timetuple()) return seconds - def from_names(self, names, force_exists=None): - """convert a list[names] to a list ZfsDatasets for this zfs_node - - Args: - :type names: list[str] - """ - ret = [] - for name in names: - ret.append(self.zfs_node.get_dataset(name, force_exists)) - - return ret # def add_virtual_snapshot(self, snapshot): # """pretend a snapshot exists (usefull in test mode)""" @@ -463,17 +457,13 @@ class ZfsDataset: :rtype: ZfsDataset """ - #FIXME: dont check for existance. (currenlty needed for _add_virtual_snapshots) - if not self.exists: - return [] - self.debug("Getting snapshots") cmd = [ "zfs", "list", "-d", "1", "-r", "-t", "snapshot", "-H", "-o", "name", self.name ] - return self.from_names(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) + return self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) @property def our_snapshots(self): @@ -572,7 +562,7 @@ class ZfsDataset: "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name ]) - return self.from_names(names[1:], force_exists=True) + return self.zfs_node.get_datasets(names[1:], force_exists=True) @CachedProperty def datasets(self, types="filesystem,volume"): @@ -588,7 +578,7 @@ class ZfsDataset: "zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name ]) - return self.from_names(names[1:], force_exists=True) + return self.zfs_node.get_datasets(names[1:], force_exists=True) def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, write_embedded, send_pipes, zfs_compressed): @@ -865,7 +855,7 @@ class ZfsDataset: snapshots = [snapshot for snapshot in self.our_snapshots if snapshot not in ignores] - return self.zfs_node.thin(snapshots, keep_objects=keeps) + return self.zfs_node.thin_list(snapshots, keep_snapshots=keeps) def thin(self, skip_holds=False): """destroys snapshots according to thin_list, except last snapshot @@ -887,11 +877,12 @@ class ZfsDataset: an initial transfer Args: + :rtype: ZfsDataset|None :type guid_check: bool :type target_dataset: ZfsDataset """ - if not target_dataset.snapshots: + if not target_dataset.exists or not target_dataset.snapshots: # target has nothing yet return None else: @@ -899,7 +890,7 @@ class ZfsDataset: target_snapshot = target_dataset.find_snapshot(source_snapshot) if target_snapshot: if guid_check and source_snapshot.properties['guid'] != target_snapshot.properties['guid']: - target_snapshot.warning("Common snapshot has invalid guid, ignoring.") + target_snapshot.warning("Common snapshots have mismatching GUID, ignoring.") else: target_snapshot.debug("common snapshot") return source_snapshot @@ -911,6 +902,7 @@ class ZfsDataset: find it. Args: + :rtype: ZfsDataset|None :type common_snapshot: ZfsDataset :type also_other_snapshots: bool """ @@ -976,27 +968,27 @@ class ZfsDataset: return allowed_filter_properties, allowed_set_properties - def _add_virtual_snapshots(self, source_dataset, source_start_snapshot, also_other_snapshots): - """add snapshots from source to our snapshot list. (just the in memory - list, no disk operations) + # def _add_virtual_snapshots(self, source_dataset, source_start_snapshot, also_other_snapshots): + # """add snapshots from source to our snapshot list. (just the in memory + # list, no disk operations) + # + # Args: + # :type source_dataset: ZfsDataset + # :type source_start_snapshot: ZfsDataset + # :type also_other_snapshots: bool + # """ + # + # self.debug("Creating virtual target snapshots") + # snapshot = source_start_snapshot + # while snapshot: + # # create virtual target snapsho + # # NOTE: with force_exist we're telling the dataset it doesnt exist yet. (e.g. its virtual) + # virtual_snapshot = self.zfs_node.get_dataset(self.filesystem_name + "@" + snapshot.snapshot_name, + # force_exists=False) + # self.snapshots.append(virtual_snapshot) + # snapshot = source_dataset.find_next_snapshot(snapshot, also_other_snapshots) - Args: - :type source_dataset: ZfsDataset - :type source_start_snapshot: ZfsDataset - :type also_other_snapshots: bool - """ - - self.debug("Creating virtual target snapshots") - snapshot = source_start_snapshot - while snapshot: - # create virtual target snapsho - # NOTE: with force_exist we're telling the dataset it doesnt exist yet. (e.g. its virtual) - virtual_snapshot = self.zfs_node.get_dataset(self.filesystem_name + "@" + snapshot.snapshot_name, - force_exists=False) - self.snapshots.append(virtual_snapshot) - snapshot = source_dataset.find_next_snapshot(snapshot, also_other_snapshots) - - def _pre_clean(self, common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_keeps): + def _pre_clean(self, common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): """cleanup old stuff before starting snapshot syncing Args: @@ -1004,7 +996,7 @@ class ZfsDataset: :type target_dataset: ZfsDataset :type source_obsoletes: list[ZfsDataset] :type target_obsoletes: list[ZfsDataset] - :type target_keeps: list[ZfsDataset] + :type target_transfers: list[ZfsDataset] """ # on source: destroy all obsoletes before common. (since we cant send them anyways) @@ -1020,7 +1012,7 @@ class ZfsDataset: # never destroy common snapshot else: target_snapshot = target_dataset.find_snapshot(source_snapshot) - if (source_snapshot in source_obsoletes) and (before_common or (target_snapshot not in target_keeps)): + if (source_snapshot in source_obsoletes) and (before_common or (target_snapshot not in target_transfers)): source_snapshot.destroy() # on target: destroy everything thats obsolete, except common_snapshot @@ -1053,7 +1045,7 @@ class ZfsDataset: return resume_token def _plan_sync(self, target_dataset, also_other_snapshots, guid_check, raw): - """plan where to start syncing and what to sync and what to keep + """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. Args: :rtype: ( ZfsDataset, ZfsDataset, list[ZfsDataset], list[ZfsDataset], list[ZfsDataset], list[ZfsDataset] ) @@ -1061,6 +1053,16 @@ class ZfsDataset: :type also_other_snapshots: bool :type guid_check: bool :type raw: bool + + Returns: + tuple: A tuple containing: + - ZfsDataset: The common snapshot + - ZfsDataset: The start snapshot + - list[ZfsDataset]: Our obsolete source snapshots, after transfer is done. (will be thinned asap) + - list[ZfsDataset]: Our obsolete target snapshots, after transfer is done. (will be thinned asap) + - list[ZfsDataset]: Transfer target snapshots. These need to be transferred. + - list[ZfsDataset]: Incompatible target snapshots. Target snapshots that are in the way, after the common snapshot. (need to be destroyed to continue) + """ # determine common and start snapshot @@ -1069,20 +1071,39 @@ class ZfsDataset: start_snapshot = self.find_start_snapshot(common_snapshot, also_other_snapshots) incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(common_snapshot, raw) - # let thinner decide whats obsolete on source + # let thinner decide whats obsolete on source after the transfer is done, keeping the last snapshot as common. source_obsoletes = [] if self.our_snapshots: source_obsoletes = self.thin_list(keeps=[self.our_snapshots[-1]])[1] - # let thinner decide keeps/obsoletes on target, AFTER the transfer would be done (by using virtual snapshots) - target_dataset._add_virtual_snapshots(self, start_snapshot, also_other_snapshots) - target_keeps = [] - target_obsoletes = [] - if target_dataset.our_snapshots: - (target_keeps, target_obsoletes) = target_dataset.thin_list(keeps=[target_dataset.our_snapshots[-1]], - ignores=incompatible_target_snapshots) + # A list of all our possible target snapshots ( existing - incompatible + transferrable from source ) + # We will use this list to let the thinner decide what to transfer to the target, and which target snapshots to destroy. - return common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_keeps, incompatible_target_snapshots + # start with snapshots that already exist, minus imcompatibles + if target_dataset.exists: + possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots if snapshot not in incompatible_target_snapshots] + else: + possible_target_snapshots = [] + + #Add all snapshots from the source to the target list, as a virtual snapshot that doesnt exist yet (force_exist=False) + source_snapshot = start_snapshot + while source_snapshot: + if also_other_snapshots or source_snapshot.is_ours(): + # virtual target snapshot + target_snapshot=target_dataset.zfs_node.get_dataset(target_dataset.filesystem_name + "@" + source_snapshot.snapshot_name, force_exists=False) + possible_target_snapshots.append(target_snapshot) + source_snapshot = self.find_next_snapshot(source_snapshot, False) + + #Now the thinner can decide which snapshots we want on the target, by looking at the whole picture: + (target_keeps, target_obsoletes)=target_dataset.zfs_node.thin_list(possible_target_snapshots, keep_snapshots=[possible_target_snapshots[-1]]) + + #Create a list of all the target snapshots we want, that don't exist yet + target_transfers=[] + for target_keep in target_keeps: + if not target_keep.exists: + target_transfers.append(target_keep) + + return common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots def handle_incompatible_snapshots(self, incompatible_target_snapshots, destroy_incompatible): """destroy incompatbile snapshots on target before sync, or inform user @@ -1147,7 +1168,8 @@ class ZfsDataset: # keep data encrypted by sending it raw (including properties) raw = True - (common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_keeps, + #note: only target_obsoletes is used during sync, to check if target doesnt want the snapshot + (common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots) = \ self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, guid_check=guid_check, raw=raw) @@ -1156,7 +1178,7 @@ class ZfsDataset: # Also usefull with no_send to still cleanup stuff. self._pre_clean( common_snapshot=common_snapshot, target_dataset=target_dataset, - target_keeps=target_keeps, target_obsoletes=target_obsoletes, source_obsoletes=source_obsoletes) + target_transfers=target_transfers, target_obsoletes=target_obsoletes, source_obsoletes=source_obsoletes) # handle incompatible stuff on target target_dataset.handle_incompatible_snapshots(incompatible_target_snapshots, destroy_incompatible) diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index 9f3fa5e..d1ebd7f 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -59,13 +59,20 @@ class ZfsNode(ExecuteNode): ExecuteNode.__init__(self, ssh_config=ssh_config, ssh_to=ssh_to, readonly=readonly, debug_output=debug_output) - def thin(self, objects, keep_objects): + def thin_list(self, snapshots, keep_snapshots): + """Returns a list of snapshots to keep and remove, according to current time and thinner settings. + + :return: ( keeps, removes ) + :type snapshots: list[ZfsDataset] + :type keep_snapshots: list[ZfsDataset] + :rtype: ( list[ZfsDataset], list[ZfsDataset] ) + """ # NOTE: if thinning is disabled with --no-thinning, self.__thinner will be none. if self.__thinner is not None: - return self.__thinner.thin(objects, keep_objects, datetime_now(self.utc).timestamp()) + return self.__thinner.thin(snapshots, keep_snapshots, datetime_now(self.utc).timestamp()) else: - return (keep_objects, []) + return (keep_snapshots, []) @CachedProperty def supported_send_options(self): @@ -110,10 +117,25 @@ class ZfsNode(ExecuteNode): return self.__pools.setdefault(zpool_name, ZfsPool(self, zpool_name)) def get_dataset(self, name, force_exists=None): - """get a ZfsDataset() object from name. stores objects internally to enable caching""" + """get a ZfsDataset() object from name. stores objects internally to enable caching + :rtype: ZfsDataset + """ return self.__datasets.setdefault(name, ZfsDataset(self, name, force_exists)) + def get_datasets(self, names, force_exists=None): + """get a list of ZfsDataset() object from names. stores objects internally to enable caching + :rtype: list[ZfsDataset] + + """ + + ret=[] + for name in names: + ret.append(self.get_dataset(name, force_exists)) + + return ret + + # def reset_progress(self): # """reset progress output counters""" # self._progress_total_bytes = 0 From 453f052a20fe83dec331741a009ae33f8a038c23 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 22 Sep 2024 15:07:56 +0200 Subject: [PATCH 002/135] fix --- zfs_autobackup/ZfsDataset.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 7536a65..92547a2 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -379,12 +379,7 @@ class ZfsDataset: def is_ours(self): """return true if this snapshot name belong to the current backup_name and snapshot formatting""" - try: - test = self.timestamp - except ValueError as e: - return False - - return True + return self.timestamp is not None @property def holds(self): @@ -417,10 +412,11 @@ class ZfsDataset: :rtype: int|None """ - if not self.is_ours(): + try: + dt = datetime.strptime(self.snapshot_name, self.zfs_node.snapshot_time_format) + except ValueError: return None - dt = datetime.strptime(self.snapshot_name, self.zfs_node.snapshot_time_format) if sys.version_info[0] >= 3: from datetime import timezone if self.zfs_node.utc: @@ -1057,7 +1053,7 @@ class ZfsDataset: Returns: tuple: A tuple containing: - ZfsDataset: The common snapshot - - ZfsDataset: The start snapshot + - ZfsDataset: The start snapshotplan_ - list[ZfsDataset]: Our obsolete source snapshots, after transfer is done. (will be thinned asap) - list[ZfsDataset]: Our obsolete target snapshots, after transfer is done. (will be thinned asap) - list[ZfsDataset]: Transfer target snapshots. These need to be transferred. From 8aa92f65929b81bd0e73b7758dbfc41738838ace Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 23 Sep 2024 22:53:01 +0200 Subject: [PATCH 003/135] wip: removed virtual snapshots, made plan_sync and sync_snapshots simpler --- zfs_autobackup/ZfsDataset.py | 149 +++++++++++++++++------------------ 1 file changed, 72 insertions(+), 77 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 92547a2..afdfd37 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -123,20 +123,20 @@ class ZfsDataset: def is_snapshot(self): """true if this dataset is a snapshot""" return self.name.find("@") != -1 - + @property def is_excluded(self): """true if this dataset is a snapshot and matches the exclude pattern""" if not self.is_snapshot: return False - + for pattern in self.zfs_node.exclude_snapshot_patterns: if pattern.search(self.name) is not None: self.debug("Excluded (path matches snapshot exclude pattern)") return True - - + + def is_selected(self, value, source, inherited, exclude_received, exclude_paths, exclude_unchanged): """determine if dataset should be selected for backup (called from @@ -450,7 +450,7 @@ class ZfsDataset: @CachedProperty def snapshots(self): """get all snapshots of this dataset - :rtype: ZfsDataset + :rtype: list[ZfsDataset] """ self.debug("Getting snapshots") @@ -471,12 +471,25 @@ class ZfsDataset: return ret + def find_snapshot_in_list(self, snapshots): + """return ZfsDataset from the list of snapshots, if it matches the snapshot_name. Otherwise None + Args: + :type snapshots: list[ZfsDataset] + :rtype: ZfsDataset|None + """ + + for snapshot in snapshots: + if snapshot.snapshot_name==self.snapshot_name: + return snapshot + + return None + def find_snapshot(self, snapshot): - """find snapshot by snapshot (can be a snapshot_name or a different - ZfsDataset ) + """find snapshot by snapshot name (can be a snapshot_name or a different + ZfsDataset) Returns None if it cant find it. Args: - :rtype: ZfsDataset + :rtype: ZfsDataset|None :type snapshot: str or ZfsDataset """ @@ -984,39 +997,31 @@ class ZfsDataset: # self.snapshots.append(virtual_snapshot) # snapshot = source_dataset.find_next_snapshot(snapshot, also_other_snapshots) - def _pre_clean(self, common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): + def _pre_clean(self, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): """cleanup old stuff before starting snapshot syncing Args: - :type common_snapshot: ZfsDataset + :type source_common_snapshot: ZfsDataset :type target_dataset: ZfsDataset :type source_obsoletes: list[ZfsDataset] :type target_obsoletes: list[ZfsDataset] :type target_transfers: list[ZfsDataset] """ - # on source: destroy all obsoletes before common. (since we cant send them anyways) - # But after common, only delete snapshots that target also doesn't want - if common_snapshot: - before_common = True - else: - before_common = False - + # on source: delete all obsoletes that are not in target_transfers (except common snapshot) for source_snapshot in self.snapshots: - if common_snapshot and source_snapshot.snapshot_name == common_snapshot.snapshot_name: - before_common = False - # never destroy common snapshot - else: - target_snapshot = target_dataset.find_snapshot(source_snapshot) - if (source_snapshot in source_obsoletes) and (before_common or (target_snapshot not in target_transfers)): - source_snapshot.destroy() + if (source_snapshot in source_obsoletes + and source_common_snapshot!=source_snapshot + and source_snapshot.find_snapshot_in_list(target_transfers) is None): + source_snapshot.destroy() # on target: destroy everything thats obsolete, except common_snapshot - for target_snapshot in target_dataset.snapshots: - if (target_snapshot in target_obsoletes) \ - and (not common_snapshot or (target_snapshot.snapshot_name != common_snapshot.snapshot_name)): - if target_snapshot.exists: - target_snapshot.destroy() + if target_dataset.exists: + for target_snapshot in target_dataset.snapshots: + if (target_snapshot in target_obsoletes) \ + and (not source_common_snapshot or (target_snapshot.snapshot_name != source_common_snapshot.snapshot_name)): + if target_snapshot.exists: + target_snapshot.destroy() def _validate_resume_token(self, target_dataset, start_snapshot): """validate and get (or destory) resume token @@ -1063,9 +1068,9 @@ class ZfsDataset: # determine common and start snapshot target_dataset.debug("Determining start snapshot") - common_snapshot = self.find_common_snapshot(target_dataset, guid_check=guid_check) - start_snapshot = self.find_start_snapshot(common_snapshot, also_other_snapshots) - incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(common_snapshot, raw) + source_common_snapshot = self.find_common_snapshot(target_dataset, guid_check=guid_check) + start_snapshot = self.find_start_snapshot(source_common_snapshot, also_other_snapshots) + incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(source_common_snapshot, raw) # let thinner decide whats obsolete on source after the transfer is done, keeping the last snapshot as common. source_obsoletes = [] @@ -1084,7 +1089,7 @@ class ZfsDataset: #Add all snapshots from the source to the target list, as a virtual snapshot that doesnt exist yet (force_exist=False) source_snapshot = start_snapshot while source_snapshot: - if also_other_snapshots or source_snapshot.is_ours(): + if (also_other_snapshots or source_snapshot.is_ours()) and not source_snapshot.is_excluded: # virtual target snapshot target_snapshot=target_dataset.zfs_node.get_dataset(target_dataset.filesystem_name + "@" + source_snapshot.snapshot_name, force_exists=False) possible_target_snapshots.append(target_snapshot) @@ -1099,7 +1104,7 @@ class ZfsDataset: if not target_keep.exists: target_transfers.append(target_keep) - return common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots + return source_common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots def handle_incompatible_snapshots(self, incompatible_target_snapshots, destroy_incompatible): """destroy incompatbile snapshots on target before sync, or inform user @@ -1165,7 +1170,7 @@ class ZfsDataset: raw = True #note: only target_obsoletes is used during sync, to check if target doesnt want the snapshot - (common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_transfers, + (source_common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots) = \ self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, guid_check=guid_check, raw=raw) @@ -1173,7 +1178,7 @@ class ZfsDataset: # NOTE: we do this because we dont want filesystems to fillup when backups keep failing. # Also usefull with no_send to still cleanup stuff. self._pre_clean( - common_snapshot=common_snapshot, target_dataset=target_dataset, + source_common_snapshot=source_common_snapshot, target_dataset=target_dataset, target_transfers=target_transfers, target_obsoletes=target_obsoletes, source_obsoletes=source_obsoletes) # handle incompatible stuff on target @@ -1196,59 +1201,49 @@ class ZfsDataset: write_embedded = False # now actually transfer the snapshots - prev_source_snapshot = common_snapshot + prev_source_snapshot = source_common_snapshot source_snapshot = start_snapshot do_rollback = rollback while source_snapshot: - target_snapshot = target_dataset.find_snapshot(source_snapshot) # still virtual + target_snapshot = source_snapshot.find_snapshot_in_list(target_transfers) - # does target actually want it? - if target_snapshot not in target_obsoletes and not source_snapshot.is_excluded: + # do the rollback, one time at first transfer + if do_rollback: + target_dataset.rollback() + do_rollback = False - # do the rollback, one time at first transfer - if do_rollback: - target_dataset.rollback() - do_rollback = False + source_snapshot.transfer_snapshot(target_snapshot, features=features, + prev_snapshot=prev_source_snapshot, show_progress=show_progress, + filter_properties=active_filter_properties, + set_properties=active_set_properties, + ignore_recv_exit_code=ignore_recv_exit_code, + resume_token=resume_token, write_embedded=write_embedded, raw=raw, + send_properties=send_properties, send_pipes=send_pipes, + recv_pipes=recv_pipes, zfs_compressed=zfs_compressed, force=force) - source_snapshot.transfer_snapshot(target_snapshot, features=features, - prev_snapshot=prev_source_snapshot, show_progress=show_progress, - filter_properties=active_filter_properties, - set_properties=active_set_properties, - ignore_recv_exit_code=ignore_recv_exit_code, - resume_token=resume_token, write_embedded=write_embedded, raw=raw, - send_properties=send_properties, send_pipes=send_pipes, - recv_pipes=recv_pipes, zfs_compressed=zfs_compressed, force=force) + resume_token = None - resume_token = None + # hold the new common snapshots and release the previous ones + if holds: + target_snapshot.hold() + source_snapshot.hold() - # hold the new common snapshots and release the previous ones + if prev_source_snapshot: if holds: - target_snapshot.hold() - source_snapshot.hold() + prev_source_snapshot.release() + target_dataset.find_snapshot(prev_source_snapshot).release() - if prev_source_snapshot: - if holds: - prev_source_snapshot.release() - target_dataset.find_snapshot(prev_source_snapshot).release() + # we may now destroy the previous source snapshot if its obsolete + if prev_source_snapshot in source_obsoletes: + prev_source_snapshot.destroy() - # we may now destroy the previous source snapshot if its obsolete - if prev_source_snapshot in source_obsoletes: - prev_source_snapshot.destroy() + # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, + # the rest was already destroyed or will not be send) + prev_target_snapshot = target_dataset.find_snapshot(prev_source_snapshot) + if prev_target_snapshot in target_obsoletes: + prev_target_snapshot.destroy() - # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, - # the rest was already destroyed or will not be send) - prev_target_snapshot = target_dataset.find_snapshot(prev_source_snapshot) - if prev_target_snapshot in target_obsoletes: - prev_target_snapshot.destroy() - - prev_source_snapshot = source_snapshot - else: - source_snapshot.debug("skipped (target doesn't need it)") - # was it actually a resume? - if resume_token: - target_dataset.verbose("Aborting resume, we dont want that snapshot anymore.") - target_dataset.abort_resume() - resume_token = None + prev_source_snapshot = source_snapshot source_snapshot = self.find_next_snapshot(source_snapshot, also_other_snapshots) From 35b079a9a949ba4cd59dbf5014c3b3f6677199a2 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Sep 2024 10:52:23 +0200 Subject: [PATCH 004/135] WIP: less hackish caching --- zfs_autobackup/ZfsDataset.py | 132 +++++++++++++++++++++++------------ zfs_autobackup/ZfsNode.py | 7 +- 2 files changed, 93 insertions(+), 46 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index afdfd37..97df5f5 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1,9 +1,9 @@ + import re from datetime import datetime import sys import time -from .CachedProperty import CachedProperty from .ExecuteNode import ExecuteError @@ -28,9 +28,30 @@ class ZfsDataset: """ self.zfs_node = zfs_node self.name = name # full name + + #caching + self.__snapshots=None #type: None|list[ZfsDataset] + self.__written_since_ours=None #type: None|int + self.__exists_check=None #type: None|bool + self.__properties=None #type: None|dict[str,str] + self.__recursive_datasets=None #type: None|list[ZfsDataset] + self.__datasets=None #type: None|list[ZfsDataset] + self.invalidate() self.force_exists = force_exists + + def invalidate(self): + """clear caches""" + # CachedProperty.clear(self) + self.force_exists = None + self.__snapshots=None + self.__written_since_ours=None + self.__exists_check=None + self.__properties=None + self.__recursive_datasets=None + self.__datasets=None + def __repr__(self): return "{}: {}".format(self.zfs_node, self.name) @@ -72,10 +93,6 @@ class ZfsDataset: """ self.zfs_node.debug("{}: {}".format(self.name, txt)) - def invalidate(self): - """clear caches""" - CachedProperty.clear(self) - self.force_exists = None def split_path(self): """return the path elements as an array""" @@ -269,13 +286,17 @@ class ZfsDataset: return self.snapshots[index] return None - @CachedProperty + @property def exists_check(self): """check on disk if it exists""" - self.debug("Checking if dataset exists") - return (len(self.zfs_node.run(tab_split=True, cmd=["zfs", "list", self.name], readonly=True, - valid_exitcodes=[0, 1], - hide_errors=True)) > 0) + + if self.__exists_check is None: + self.debug("Checking if dataset exists") + self.__exists_check=(len(self.zfs_node.run(tab_split=True, cmd=["zfs", "list", self.name], readonly=True, + valid_exitcodes=[0, 1], + hide_errors=True)) > 0) + + return self.__exists_check @property def exists(self): @@ -344,22 +365,24 @@ class ZfsDataset: else: raise - @CachedProperty + @property def properties(self): """all zfs properties""" - cmd = [ - "zfs", "get", "-H", "-o", "property,value", "-p", "all", self.name - ] + if self.__properties is None: - self.debug("Getting zfs properties") + cmd = [ + "zfs", "get", "-H", "-o", "property,value", "-p", "all", self.name + ] - ret = {} - for pair in self.zfs_node.run(tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]): - if len(pair) == 2: - ret[pair[0]] = pair[1] + self.debug("Getting zfs properties") - return ret + self.__properties = {} + for pair in self.zfs_node.run(tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]): + if len(pair) == 2: + self.__properties[pair[0]] = pair[1] + + return self.__properties def is_changed(self, min_changed_bytes=1): """dataset is changed since ANY latest snapshot ? @@ -447,24 +470,32 @@ class ZfsDataset: # print ("ADDED VIRT") # self._virtual_snapshots.append(snapshot) - @CachedProperty + @property def snapshots(self): """get all snapshots of this dataset :rtype: list[ZfsDataset] """ - self.debug("Getting snapshots") + #cached? + if self.__snapshots is None: - cmd = [ - "zfs", "list", "-d", "1", "-r", "-t", "snapshot", "-H", "-o", "name", self.name - ] - return self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) + self.debug("Getting snapshots") + + cmd = [ + "zfs", "list", "-d", "1", "-r", "-t", "snapshot", "-H", "-o", "name", self.name + ] + + self.__snapshots=self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) + + + return self.__snapshots @property def our_snapshots(self): """get list[snapshots] creates by us of this dataset""" ret = [] + for snapshot in self.snapshots: if snapshot.is_ours(): ret.append(snapshot) @@ -525,18 +556,22 @@ class ZfsDataset: return None - @CachedProperty + @property def written_since_ours(self): """get number of bytes written since our last snapshot""" - latest_snapshot = self.our_snapshots[-1] + if self.__written_since_ours is None: - self.debug("Getting bytes written since our last snapshot") - cmd = ["zfs", "get", "-H", "-ovalue", "-p", "written@" + str(latest_snapshot), self.name] + latest_snapshot = self.our_snapshots[-1] - output = self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0]) + self.debug("Getting bytes written since our last snapshot") + cmd = ["zfs", "get", "-H", "-ovalue", "-p", "written@" + str(latest_snapshot), self.name] - return int(output[0]) + output = self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0]) + + self.__written_since_ours=int(output[0]) + + return self.__written_since_ours def is_changed_ours(self, min_changed_bytes=1): """dataset is changed since OUR latest snapshot? @@ -557,23 +592,28 @@ class ZfsDataset: return True - @CachedProperty + @property def recursive_datasets(self, types="filesystem,volume"): """get all (non-snapshot) datasets recursively under us Args: :type types: str + :rtype: list[ZfsDataset] """ - self.debug("Getting all recursive datasets under us") + if self.__recursive_datasets is None: - names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ - "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name - ]) + self.debug("Getting all recursive datasets under us") - return self.zfs_node.get_datasets(names[1:], force_exists=True) + names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ + "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name + ]) - @CachedProperty + self.__recursive_datasets=self.zfs_node.get_datasets(names[1:], force_exists=True) + + return self.__recursive_datasets + + @property def datasets(self, types="filesystem,volume"): """get all (non-snapshot) datasets directly under us @@ -581,13 +621,17 @@ class ZfsDataset: :type types: str """ - self.debug("Getting all datasets under us") + if self.__datasets is None: - names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ - "zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name - ]) + self.debug("Getting all datasets under us") - return self.zfs_node.get_datasets(names[1:], force_exists=True) + names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ + "zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name + ]) + + self.__datasets=self.zfs_node.get_datasets(names[1:], force_exists=True) + + return self.__datasets def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, write_embedded, send_pipes, zfs_compressed): diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index d1ebd7f..b94651b 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -209,6 +209,8 @@ class ZfsNode(ExecuteNode): def consistent_snapshot(self, datasets, snapshot_name, min_changed_bytes, pre_snapshot_cmds=[], post_snapshot_cmds=[], set_snapshot_properties=[]): """create a consistent (atomic) snapshot of specified datasets, per pool. + Args: + :type datasets: list[ZfsDataset] """ pools = {} @@ -230,8 +232,9 @@ class ZfsNode(ExecuteNode): pools[pool].append(snapshot) # update cache, but try to prevent an unneeded zfs list - if self.readonly or CachedProperty.is_cached(dataset, 'snapshots'): - dataset.snapshots.append(snapshot) # NOTE: this will trigger zfs list if its not cached + # if self.readonly or CachedProperty.is_cached(dataset, 'snapshots'): + #dataset.snapshots.append(snapshot) # NOTE: this will trigger zfs list if its not cached + dataset.invalidate() if not pools: self.verbose("No changes anywhere: not creating snapshots.") From 9a48851166a78e585ec7cb8de6e130281cdba93c Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Sep 2024 11:53:27 +0200 Subject: [PATCH 005/135] WIP: fixes. Also assume the v3.4 branch is broken for a while since im doing heavy refactoring --- tests/test_zfsautobackup.py | 2 +- zfs_autobackup/ZfsDataset.py | 34 ++++++++++++++++++++++++++-------- zfs_autobackup/ZfsNode.py | 5 +---- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index f05f15e..142224b 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -823,7 +823,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 #make another backup but with no-holds. we should naturally endup with only number 3 with mocktime("20101111000003"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --keep-source=0 --keep-target=0 --no-holds --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --keep-source=0 --keep-target=0 --no-holds --allow-empty --debug".split(" ")).run()) r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) self.assertMultiLineEqual(r, """ diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 97df5f5..a5adac5 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -491,6 +491,15 @@ class ZfsDataset: return self.__snapshots + def cache_snapshot(self, snapshot): + """Update our snapshot cache (if we have any) + Args: + :type snapshot: ZfsDataset + """ + + if self.__snapshots is not None: + self.__snapshots.append(snapshot) + @property def our_snapshots(self): """get list[snapshots] creates by us of this dataset""" @@ -521,9 +530,12 @@ class ZfsDataset: Args: :rtype: ZfsDataset|None - :type snapshot: str or ZfsDataset + :type snapshot: str|ZfsDataset|None """ + if snapshot is None: + return None + if not isinstance(snapshot, ZfsDataset): snapshot_name = snapshot else: @@ -1140,7 +1152,11 @@ class ZfsDataset: source_snapshot = self.find_next_snapshot(source_snapshot, False) #Now the thinner can decide which snapshots we want on the target, by looking at the whole picture: - (target_keeps, target_obsoletes)=target_dataset.zfs_node.thin_list(possible_target_snapshots, keep_snapshots=[possible_target_snapshots[-1]]) + if possible_target_snapshots: + (target_keeps, target_obsoletes)=target_dataset.zfs_node.thin_list(possible_target_snapshots, keep_snapshots=[possible_target_snapshots[-1]]) + else: + target_keeps = [] + target_obsoletes = [] #Create a list of all the target snapshots we want, that don't exist yet target_transfers=[] @@ -1213,13 +1229,12 @@ class ZfsDataset: # keep data encrypted by sending it raw (including properties) raw = True - #note: only target_obsoletes is used during sync, to check if target doesnt want the snapshot (source_common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots) = \ self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, guid_check=guid_check, raw=raw) - # NOTE: we do this because we dont want filesystems to fillup when backups keep failing. + # NOTE: we do a pre-clean because we dont want filesystems to fillup when backups keep failing. # Also usefull with no_send to still cleanup stuff. self._pre_clean( source_common_snapshot=source_common_snapshot, target_dataset=target_dataset, @@ -1246,6 +1261,8 @@ class ZfsDataset: # now actually transfer the snapshots prev_source_snapshot = source_common_snapshot + prev_target_snapshot=target_dataset.find_snapshot(prev_source_snapshot) + source_snapshot = start_snapshot do_rollback = rollback while source_snapshot: @@ -1272,10 +1289,11 @@ class ZfsDataset: target_snapshot.hold() source_snapshot.hold() - if prev_source_snapshot: - if holds: + if prev_source_snapshot: prev_source_snapshot.release() - target_dataset.find_snapshot(prev_source_snapshot).release() + + if prev_target_snapshot: + prev_target_snapshot.release() # we may now destroy the previous source snapshot if its obsolete if prev_source_snapshot in source_obsoletes: @@ -1283,11 +1301,11 @@ class ZfsDataset: # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, # the rest was already destroyed or will not be send) - prev_target_snapshot = target_dataset.find_snapshot(prev_source_snapshot) if prev_target_snapshot in target_obsoletes: prev_target_snapshot.destroy() prev_source_snapshot = source_snapshot + prev_target_snapshot = target_snapshot source_snapshot = self.find_next_snapshot(source_snapshot, also_other_snapshots) diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index b94651b..b7640e9 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -231,10 +231,7 @@ class ZfsNode(ExecuteNode): pools[pool].append(snapshot) - # update cache, but try to prevent an unneeded zfs list - # if self.readonly or CachedProperty.is_cached(dataset, 'snapshots'): - #dataset.snapshots.append(snapshot) # NOTE: this will trigger zfs list if its not cached - dataset.invalidate() + dataset.cache_snapshot(snapshot) if not pools: self.verbose("No changes anywhere: not creating snapshots.") From 6b987b24602df62a1504cde76620c7a9e9e149e8 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Sep 2024 13:58:01 +0200 Subject: [PATCH 006/135] cleaned up planner --- zfs_autobackup/ZfsDataset.py | 101 ++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index a5adac5..fdb9358 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -268,12 +268,11 @@ class ZfsDataset: # return self.snapshots[index] # return None - def find_next_snapshot(self, snapshot, also_other_snapshots=False): + def find_next_snapshot(self, snapshot): """find next snapshot in this dataset. None if it doesn't exist Args: :type snapshot: ZfsDataset - :type also_other_snapshots: bool """ if self.is_snapshot: @@ -282,8 +281,7 @@ class ZfsDataset: index = self.find_snapshot_index(snapshot) while index is not None and index < len(self.snapshots) - 1: index = index + 1 - if also_other_snapshots or self.snapshots[index].is_ours(): - return self.snapshots[index] + return self.snapshots[index] return None @property @@ -962,31 +960,31 @@ class ZfsDataset: # target_dataset.error("Cant find common snapshot with source.") raise (Exception("Cant find common snapshot with target.")) - def find_start_snapshot(self, common_snapshot, also_other_snapshots): - """finds first snapshot to send :rtype: ZfsDataset or None if we cant - find it. - - Args: - :rtype: ZfsDataset|None - :type common_snapshot: ZfsDataset - :type also_other_snapshots: bool - """ - - if not common_snapshot: - if not self.snapshots: - start_snapshot = None - else: - # no common snapshot, start from beginning - start_snapshot = self.snapshots[0] - - if not start_snapshot.is_ours() and not also_other_snapshots: - # try to start at a snapshot thats ours - start_snapshot = self.find_next_snapshot(start_snapshot, also_other_snapshots) - else: - # normal situation: start_snapshot is the one after the common snapshot - start_snapshot = self.find_next_snapshot(common_snapshot, also_other_snapshots) - - return start_snapshot + # def find_start_snapshot(self, common_snapshot, also_other_snapshots): + # """finds first snapshot to send :rtype: ZfsDataset or None if we cant + # find it. + # + # Args: + # :rtype: ZfsDataset|None + # :type common_snapshot: ZfsDataset + # :type also_other_snapshots: bool + # """ + # + # if not common_snapshot: + # if not self.snapshots: + # start_snapshot = None + # else: + # # no common snapshot, start from beginning + # start_snapshot = self.snapshots[0] + # + # if not start_snapshot.is_ours() and not also_other_snapshots: + # # try to start at a snapshot thats ours + # start_snapshot = self.find_next_snapshot(start_snapshot, also_other_snapshots) + # else: + # # normal situation: start_snapshot is the one after the common snapshot + # start_snapshot = self.find_next_snapshot(common_snapshot, also_other_snapshots) + # + # return start_snapshot def find_incompatible_snapshots(self, common_snapshot, raw): """returns a list[snapshots] that is incompatible for a zfs recv onto @@ -1088,7 +1086,7 @@ class ZfsDataset: """ if target_dataset.exists and 'receive_resume_token' in target_dataset.properties: - if start_snapshot == None: + if start_snapshot is None: target_dataset.verbose("Aborting resume, its obsolete.") target_dataset.abort_resume() else: @@ -1105,7 +1103,7 @@ class ZfsDataset: """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. Args: - :rtype: ( ZfsDataset, ZfsDataset, list[ZfsDataset], list[ZfsDataset], list[ZfsDataset], list[ZfsDataset] ) + :rtype: ( ZfsDataset, list[ZfsDataset], list[ZfsDataset], list[ZfsDataset], list[ZfsDataset] ) :type target_dataset: ZfsDataset :type also_other_snapshots: bool :type guid_check: bool @@ -1114,7 +1112,6 @@ class ZfsDataset: Returns: tuple: A tuple containing: - ZfsDataset: The common snapshot - - ZfsDataset: The start snapshotplan_ - list[ZfsDataset]: Our obsolete source snapshots, after transfer is done. (will be thinned asap) - list[ZfsDataset]: Our obsolete target snapshots, after transfer is done. (will be thinned asap) - list[ZfsDataset]: Transfer target snapshots. These need to be transferred. @@ -1125,7 +1122,7 @@ class ZfsDataset: # determine common and start snapshot target_dataset.debug("Determining start snapshot") source_common_snapshot = self.find_common_snapshot(target_dataset, guid_check=guid_check) - start_snapshot = self.find_start_snapshot(source_common_snapshot, also_other_snapshots) + # start_snapshot = self.find_start_snapshot(source_common_snapshot, also_other_snapshots) incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(source_common_snapshot, raw) # let thinner decide whats obsolete on source after the transfer is done, keeping the last snapshot as common. @@ -1133,23 +1130,25 @@ class ZfsDataset: if self.our_snapshots: source_obsoletes = self.thin_list(keeps=[self.our_snapshots[-1]])[1] - # A list of all our possible target snapshots ( existing - incompatible + transferrable from source ) - # We will use this list to let the thinner decide what to transfer to the target, and which target snapshots to destroy. - # start with snapshots that already exist, minus imcompatibles if target_dataset.exists: possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots if snapshot not in incompatible_target_snapshots] else: possible_target_snapshots = [] - #Add all snapshots from the source to the target list, as a virtual snapshot that doesnt exist yet (force_exist=False) - source_snapshot = start_snapshot + #Add all snapshots from the source, starting after the common snapshot if it exists + if source_common_snapshot: + source_snapshot=self.find_next_snapshot(source_common_snapshot ) + else: + source_snapshot=self.snapshots[0] + while source_snapshot: + # we want it? if (also_other_snapshots or source_snapshot.is_ours()) and not source_snapshot.is_excluded: - # virtual target snapshot + # create virtual target snapshot target_snapshot=target_dataset.zfs_node.get_dataset(target_dataset.filesystem_name + "@" + source_snapshot.snapshot_name, force_exists=False) possible_target_snapshots.append(target_snapshot) - source_snapshot = self.find_next_snapshot(source_snapshot, False) + source_snapshot = self.find_next_snapshot(source_snapshot) #Now the thinner can decide which snapshots we want on the target, by looking at the whole picture: if possible_target_snapshots: @@ -1164,7 +1163,7 @@ class ZfsDataset: if not target_keep.exists: target_transfers.append(target_keep) - return source_common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots + return source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots def handle_incompatible_snapshots(self, incompatible_target_snapshots, destroy_incompatible): """destroy incompatbile snapshots on target before sync, or inform user @@ -1229,7 +1228,7 @@ class ZfsDataset: # keep data encrypted by sending it raw (including properties) raw = True - (source_common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_transfers, + (source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots) = \ self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, guid_check=guid_check, raw=raw) @@ -1248,7 +1247,7 @@ class ZfsDataset: return # check if we can resume - resume_token = self._validate_resume_token(target_dataset, start_snapshot) + resume_token = self._validate_resume_token(target_dataset, target_transfers[0]) (active_filter_properties, active_set_properties) = self.get_allowed_properties(filter_properties, set_properties) @@ -1260,13 +1259,17 @@ class ZfsDataset: write_embedded = False # now actually transfer the snapshots - prev_source_snapshot = source_common_snapshot - prev_target_snapshot=target_dataset.find_snapshot(prev_source_snapshot) + # prev_source_snapshot = start_snapshot + # source_dataset=start_snapshot.parent + # prev_target_snapshot=target_dataset.find_snapshot(prev_source_snapshot) - source_snapshot = start_snapshot do_rollback = rollback - while source_snapshot: - target_snapshot = source_snapshot.find_snapshot_in_list(target_transfers) + prev_source_snapshot=source_common_snapshot + prev_target_snapshot=target_dataset.find_snapshot(source_common_snapshot) + for target_snapshot in target_transfers: + + source_snapshot=self.find_snapshot(target_snapshot) + # do the rollback, one time at first transfer if do_rollback: @@ -1307,7 +1310,7 @@ class ZfsDataset: prev_source_snapshot = source_snapshot prev_target_snapshot = target_snapshot - source_snapshot = self.find_next_snapshot(source_snapshot, also_other_snapshots) + # source_snapshot = self.find_next_snapshot(source_snapshot, also_other_snapshots) def mount(self, mount_point): From cc1e7c24b11109f3c62c836e221d639d4b16c6b1 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Sep 2024 15:14:41 +0200 Subject: [PATCH 007/135] fix --- tests/test_zfsautobackup.py | 2 +- zfs_autobackup/ZfsDataset.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 142224b..0a60477 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -549,7 +549,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 canmount - - with mocktime("20101111000003"): #--test should succeed by destroying incompatibles - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --destroy-incompatible --test".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --destroy-incompatible --test --debug".split(" ")).run()) with mocktime("20101111000003"): #should succeed by destroying incompatibles diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index fdb9358..872c0f3 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1140,7 +1140,10 @@ class ZfsDataset: if source_common_snapshot: source_snapshot=self.find_next_snapshot(source_common_snapshot ) else: - source_snapshot=self.snapshots[0] + if self.snapshots: + source_snapshot=self.snapshots[0] + else: + source_snapshot=None while source_snapshot: # we want it? @@ -1243,7 +1246,7 @@ class ZfsDataset: target_dataset.handle_incompatible_snapshots(incompatible_target_snapshots, destroy_incompatible) # now actually transfer the snapshots, if we want - if no_send: + if no_send or len(target_transfers)==0: return # check if we can resume @@ -1270,7 +1273,6 @@ class ZfsDataset: source_snapshot=self.find_snapshot(target_snapshot) - # do the rollback, one time at first transfer if do_rollback: target_dataset.rollback() From a6680ee20571254b71f1b63e255c7385899cf61e Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Sep 2024 15:21:31 +0200 Subject: [PATCH 008/135] --- zfs_autobackup/ZfsDataset.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 872c0f3..46bcdae 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1119,10 +1119,9 @@ class ZfsDataset: """ - # determine common and start snapshot + ### 1: determine common and start snapshot target_dataset.debug("Determining start snapshot") source_common_snapshot = self.find_common_snapshot(target_dataset, guid_check=guid_check) - # start_snapshot = self.find_start_snapshot(source_common_snapshot, also_other_snapshots) incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(source_common_snapshot, raw) # let thinner decide whats obsolete on source after the transfer is done, keeping the last snapshot as common. @@ -1130,13 +1129,15 @@ class ZfsDataset: if self.our_snapshots: source_obsoletes = self.thin_list(keeps=[self.our_snapshots[-1]])[1] + ### 2: Determine possible target snapshots + # start with snapshots that already exist, minus imcompatibles if target_dataset.exists: possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots if snapshot not in incompatible_target_snapshots] else: possible_target_snapshots = [] - #Add all snapshots from the source, starting after the common snapshot if it exists + # add all snapshots from the source, starting after the common snapshot if it exists if source_common_snapshot: source_snapshot=self.find_next_snapshot(source_common_snapshot ) else: @@ -1153,14 +1154,14 @@ class ZfsDataset: possible_target_snapshots.append(target_snapshot) source_snapshot = self.find_next_snapshot(source_snapshot) - #Now the thinner can decide which snapshots we want on the target, by looking at the whole picture: + ### 3: Let the thinner decide what it wants by looking at all the possible target_snaphots at once if possible_target_snapshots: (target_keeps, target_obsoletes)=target_dataset.zfs_node.thin_list(possible_target_snapshots, keep_snapshots=[possible_target_snapshots[-1]]) else: target_keeps = [] target_obsoletes = [] - #Create a list of all the target snapshots we want, that don't exist yet + ### 4: Look at what the thinner wants and create a list of snapshots we still need to transfer target_transfers=[] for target_keep in target_keeps: if not target_keep.exists: From f51ef2a909414bb2ebfc2803f2701c8ea556f213 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Sep 2024 15:25:23 +0200 Subject: [PATCH 009/135] test fix --- tests/test_externalfailures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_externalfailures.py b/tests/test_externalfailures.py index 7c1dcd1..5b55c53 100644 --- a/tests/test_externalfailures.py +++ b/tests/test_externalfailures.py @@ -204,7 +204,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 print(buf.getvalue()) - self.assertIn("Aborting resume, we dont want that snapshot anymore.", buf.getvalue()) + self.assertIn("Aborting resume", buf.getvalue()) r = shelltest("zfs list -H -o name -r -t all test_target1") self.assertMultiLineEqual(r, """ From 1751db92b04966493cf314cc0cdbc715242e1aec Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 24 Sep 2024 18:07:56 +0200 Subject: [PATCH 010/135] more test fixes --- tests/test_externalfailures.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_externalfailures.py b/tests/test_externalfailures.py index 5b55c53..05f25a0 100644 --- a/tests/test_externalfailures.py +++ b/tests/test_externalfailures.py @@ -240,9 +240,9 @@ test_target1/test_source2/fs2/sub@test-20101111000002 "test test_target1 --no-progress --verbose --no-snapshot".split( " ")).run()) - print(buf.getvalue()) - - self.assertIn("Aborting resume, its obsolete", buf.getvalue()) + #NOTE: v3.4 doesnt abort the resume if there are no snapshots to send. + # print(buf.getvalue()) + # self.assertIn("Aborting resume, its obsolete", buf.getvalue()) def test_missing_common(self): From 682ad53d1ed5007bd3aea0f8dd70f61fcdd6654e Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 24 Sep 2024 18:14:53 +0200 Subject: [PATCH 011/135] remove commented stuff --- zfs_autobackup/ZfsDataset.py | 47 ------------------------------------ 1 file changed, 47 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 46bcdae..0e2a326 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -960,31 +960,6 @@ class ZfsDataset: # target_dataset.error("Cant find common snapshot with source.") raise (Exception("Cant find common snapshot with target.")) - # def find_start_snapshot(self, common_snapshot, also_other_snapshots): - # """finds first snapshot to send :rtype: ZfsDataset or None if we cant - # find it. - # - # Args: - # :rtype: ZfsDataset|None - # :type common_snapshot: ZfsDataset - # :type also_other_snapshots: bool - # """ - # - # if not common_snapshot: - # if not self.snapshots: - # start_snapshot = None - # else: - # # no common snapshot, start from beginning - # start_snapshot = self.snapshots[0] - # - # if not start_snapshot.is_ours() and not also_other_snapshots: - # # try to start at a snapshot thats ours - # start_snapshot = self.find_next_snapshot(start_snapshot, also_other_snapshots) - # else: - # # normal situation: start_snapshot is the one after the common snapshot - # start_snapshot = self.find_next_snapshot(common_snapshot, also_other_snapshots) - # - # return start_snapshot def find_incompatible_snapshots(self, common_snapshot, raw): """returns a list[snapshots] that is incompatible for a zfs recv onto @@ -1031,25 +1006,6 @@ class ZfsDataset: return allowed_filter_properties, allowed_set_properties - # def _add_virtual_snapshots(self, source_dataset, source_start_snapshot, also_other_snapshots): - # """add snapshots from source to our snapshot list. (just the in memory - # list, no disk operations) - # - # Args: - # :type source_dataset: ZfsDataset - # :type source_start_snapshot: ZfsDataset - # :type also_other_snapshots: bool - # """ - # - # self.debug("Creating virtual target snapshots") - # snapshot = source_start_snapshot - # while snapshot: - # # create virtual target snapsho - # # NOTE: with force_exist we're telling the dataset it doesnt exist yet. (e.g. its virtual) - # virtual_snapshot = self.zfs_node.get_dataset(self.filesystem_name + "@" + snapshot.snapshot_name, - # force_exists=False) - # self.snapshots.append(virtual_snapshot) - # snapshot = source_dataset.find_next_snapshot(snapshot, also_other_snapshots) def _pre_clean(self, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): """cleanup old stuff before starting snapshot syncing @@ -1263,9 +1219,6 @@ class ZfsDataset: write_embedded = False # now actually transfer the snapshots - # prev_source_snapshot = start_snapshot - # source_dataset=start_snapshot.parent - # prev_target_snapshot=target_dataset.find_snapshot(prev_source_snapshot) do_rollback = rollback prev_source_snapshot=source_common_snapshot From 3cbc93e8ccc9f3277cc14f8d57ed338d20c382de Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 24 Sep 2024 18:17:33 +0200 Subject: [PATCH 012/135] only invalidate __properties --- zfs_autobackup/ZfsDataset.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 0e2a326..049fbc6 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1312,7 +1312,8 @@ class ZfsDataset: self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - self.invalidate() + #invalidate cache + self.__properties=None def inherit(self, prop): """inherit zfs property""" @@ -1325,4 +1326,5 @@ class ZfsDataset: self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - self.invalidate() + #invalidate cache + self.__properties=None From 5f2e686a1b38a00afe23ee21a476b2a993014b89 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 24 Sep 2024 18:18:24 +0200 Subject: [PATCH 013/135] only invalidate __properties --- zfs_autobackup/ZfsDataset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 049fbc6..9efa79d 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -37,11 +37,11 @@ class ZfsDataset: self.__recursive_datasets=None #type: None|list[ZfsDataset] self.__datasets=None #type: None|list[ZfsDataset] - self.invalidate() + self.invalidate_cache() self.force_exists = force_exists - def invalidate(self): + def invalidate_cache(self): """clear caches""" # CachedProperty.clear(self) self.force_exists = None @@ -354,7 +354,7 @@ class ZfsDataset: else: self.zfs_node.run(["zfs", "destroy", self.name]) - self.invalidate() + self.invalidate_cache() self.force_exists = False return True except ExecuteError: @@ -766,7 +766,7 @@ class ZfsDataset: self.zfs_node.run(cmd, inp=pipe, valid_exitcodes=valid_exitcodes) # invalidate cache - self.invalidate() + self.invalidate_cache() # in test mode we assume everything was ok and it exists if self.zfs_node.readonly: From 07542365ac2c29da03399d099e37a0f80547e5da Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 24 Sep 2024 18:22:17 +0200 Subject: [PATCH 014/135] reformatting. changed version number --- zfs_autobackup/BlockHasher.py | 32 +++++----- zfs_autobackup/CachedProperty.py | 2 +- zfs_autobackup/CliBase.py | 23 ++++--- zfs_autobackup/CmdPipe.py | 7 +-- zfs_autobackup/ExecuteNode.py | 55 +++++++++-------- zfs_autobackup/LogConsole.py | 11 ++-- zfs_autobackup/LogStub.py | 6 +- zfs_autobackup/Thinner.py | 6 +- zfs_autobackup/ThinnerRule.py | 2 +- zfs_autobackup/TreeHasher.py | 27 ++++---- zfs_autobackup/ZfsAuto.py | 41 +++++++------ zfs_autobackup/ZfsAutobackup.py | 31 +++++----- zfs_autobackup/ZfsAutoverify.py | 59 +++++++++--------- zfs_autobackup/ZfsCheck.py | 75 ++++++++++++----------- zfs_autobackup/ZfsDataset.py | 102 ++++++++++++++----------------- zfs_autobackup/ZfsPool.py | 2 +- zfs_autobackup/compressors.py | 37 +++++------ zfs_autobackup/util.py | 26 ++++---- 18 files changed, 268 insertions(+), 276 deletions(-) diff --git a/zfs_autobackup/BlockHasher.py b/zfs_autobackup/BlockHasher.py index 0a37c90..0e82f59 100644 --- a/zfs_autobackup/BlockHasher.py +++ b/zfs_autobackup/BlockHasher.py @@ -19,15 +19,14 @@ class BlockHasher(): def __init__(self, count=10000, bs=4096, hash_class=hashlib.sha1, skip=0): self.count = count self.bs = bs - self.chunk_size=bs*count + self.chunk_size = bs * count self.hash_class = hash_class # self.coverage=coverage - self.skip=skip - self._skip_count=0 - - self.stats_total_bytes=0 + self.skip = skip + self._skip_count = 0 + self.stats_total_bytes = 0 def _seek_next_chunk(self, fh, fsize): """seek fh to next chunk and update skip counter. @@ -37,8 +36,8 @@ class BlockHasher(): """ - #ignore rempty files - if fsize==0: + # ignore rempty files + if fsize == 0: return False # need to skip chunks? @@ -53,7 +52,7 @@ class BlockHasher(): # seek to next chunk, reset skip count fh.seek(self.chunk_size * self._skip_count, os.SEEK_CUR) self._skip_count = self.skip - return fh.tell()//self.chunk_size + return fh.tell() // self.chunk_size else: # should read this chunk, reset skip count self._skip_count = self.skip @@ -67,24 +66,23 @@ class BlockHasher(): yields nothing for empty files. """ - with open(fname, "rb") as fh: fh.seek(0, os.SEEK_END) - fsize=fh.tell() + fsize = fh.tell() fh.seek(0) - while fh.tell() {}".format(exit_code)) if (valid_exitcodes != []) and (exit_code not in valid_exitcodes): - self.error("Command \"{}\" returned exit code {} (valid codes: {})".format(cmd_item, exit_code, valid_exitcodes)) + self.error("Command \"{}\" returned exit code {} (valid codes: {})".format(cmd_item, exit_code, + valid_exitcodes)) return False return True @@ -149,7 +150,7 @@ class ExecuteNode(LogStub): if pipe: # dont specify output handler, so it will get piped to next process - stdout_handler=None + stdout_handler = None else: # handle output manually, dont pipe it def stdout_handler(line): @@ -160,7 +161,8 @@ class ExecuteNode(LogStub): self._parse_stdout(line) # add shell command and handlers to pipe - cmd_item=CmdItem(cmd=self._shell_cmd(cmd, cwd), readonly=readonly, stderr_handler=stderr_handler, exit_handler=exit_handler, shell=self.is_local(), stdout_handler=stdout_handler) + cmd_item = CmdItem(cmd=self._shell_cmd(cmd, cwd), readonly=readonly, stderr_handler=stderr_handler, + exit_handler=exit_handler, shell=self.is_local(), stdout_handler=stdout_handler) cmd_pipe.add(cmd_item) # return CmdPipe instead of executing? @@ -174,7 +176,7 @@ class ExecuteNode(LogStub): # execute and calls handlers in CmdPipe if not cmd_pipe.execute(): - raise(ExecuteError("Last command returned error")) + raise (ExecuteError("Last command returned error")) if return_all: return output_lines, error_lines, cmd_item.process and cmd_item.process.returncode @@ -183,7 +185,8 @@ class ExecuteNode(LogStub): else: return output_lines - def script(self, lines, inp=None, stdout_handler=None, stderr_handler=None, exit_handler=None, valid_exitcodes=None, readonly=False, hide_errors=False, pipe=False): + def script(self, lines, inp=None, stdout_handler=None, stderr_handler=None, exit_handler=None, valid_exitcodes=None, + readonly=False, hide_errors=False, pipe=False): """Run a multiline script on the node. This is much more low level than run() and allows for finer grained control. @@ -212,14 +215,14 @@ class ExecuteNode(LogStub): # add stuff to existing pipe cmd_pipe = inp - internal_stdout_handler=None + internal_stdout_handler = None if stdout_handler is not None: if self.debug_output: def internal_stdout_handler(line): self.debug("STDOUT > " + line.rstrip()) stdout_handler(line) else: - internal_stdout_handler=stdout_handler + internal_stdout_handler = stdout_handler def internal_stderr_handler(line): self._parse_stderr(line, hide_errors) @@ -243,12 +246,12 @@ class ExecuteNode(LogStub): return True - #build command - cmd=[] + # build command + cmd = [] - #add remote shell + # add remote shell if not self.is_local(): - #note: dont escape this part (executed directly without shell) + # note: dont escape this part (executed directly without shell) cmd.append("ssh") if self.ssh_config is not None: @@ -260,7 +263,9 @@ class ExecuteNode(LogStub): cmd.append("\n".join(lines)) # add shell command and handlers to pipe - cmd_item=CmdItem(cmd=cmd, readonly=readonly, stderr_handler=internal_stderr_handler, exit_handler=internal_exit_handler, stdout_handler=internal_stdout_handler, shell=self.is_local()) + cmd_item = CmdItem(cmd=cmd, readonly=readonly, stderr_handler=internal_stderr_handler, + exit_handler=internal_exit_handler, stdout_handler=internal_stdout_handler, + shell=self.is_local()) cmd_pipe.add(cmd_item) self.debug("SCRIPT > {}".format(cmd_pipe)) diff --git a/zfs_autobackup/LogConsole.py b/zfs_autobackup/LogConsole.py index a7e799a..ebec45e 100644 --- a/zfs_autobackup/LogConsole.py +++ b/zfs_autobackup/LogConsole.py @@ -3,6 +3,7 @@ from __future__ import print_function import sys + class LogConsole: """Log-class that outputs to console, adding colors if needed""" @@ -10,11 +11,11 @@ class LogConsole: self.last_log = "" self.show_debug = show_debug self.show_verbose = show_verbose - self._progress_uncleared=False + self._progress_uncleared = False if color: # try to use color, failback if colorama not available - self.colorama=False + self.colorama = False try: import colorama global colorama @@ -23,7 +24,7 @@ class LogConsole: pass else: - self.colorama=False + self.colorama = False def error(self, txt): self.clear_progress() @@ -62,7 +63,7 @@ class LogConsole: def progress(self, txt): """print progress output to stderr (stays on same line)""" self.clear_progress() - self._progress_uncleared=True + self._progress_uncleared = True print(">>> {}\r".format(txt), end='', file=sys.stderr) sys.stderr.flush() @@ -71,4 +72,4 @@ class LogConsole: import colorama print(colorama.ansi.clear_line(), end='', file=sys.stderr) # sys.stderr.flush() - self._progress_uncleared=False + self._progress_uncleared = False diff --git a/zfs_autobackup/LogStub.py b/zfs_autobackup/LogStub.py index 5d593c1..0d00e31 100644 --- a/zfs_autobackup/LogStub.py +++ b/zfs_autobackup/LogStub.py @@ -1,5 +1,5 @@ -#Used for baseclasses that dont implement their own logging (Like ExecuteNode) -#Usually logging is implemented in subclasses (Like ZfsNode thats a subclass of ExecuteNode), but for regression testing its nice to have these stubs. +# Used for baseclasses that dont implement their own logging (Like ExecuteNode) +# Usually logging is implemented in subclasses (Like ZfsNode thats a subclass of ExecuteNode), but for regression testing its nice to have these stubs. class LogStub: """Just a stub, usually overriden in subclasses.""" @@ -15,4 +15,4 @@ class LogStub: print("WARNING: " + txt) def error(self, txt): - print("ERROR : " + txt) \ No newline at end of file + print("ERROR : " + txt) diff --git a/zfs_autobackup/Thinner.py b/zfs_autobackup/Thinner.py index 1cbb3ea..ec7a059 100644 --- a/zfs_autobackup/Thinner.py +++ b/zfs_autobackup/Thinner.py @@ -1,4 +1,3 @@ - from .ThinnerRule import ThinnerRule @@ -48,7 +47,6 @@ class Thinner: now: if specified, use this time as current time """ - # always keep a number of the last objets? if self.always_keep: # all of them @@ -71,7 +69,7 @@ class Thinner: # traverse objects for thisobject in objects: - #ignore stuff without timestamp, always keep those. + # ignore stuff without timestamp, always keep those. if thisobject.timestamp is None: keeps.append(thisobject) else: @@ -96,4 +94,4 @@ class Thinner: else: removes.append(thisobject) - return keeps, removes \ No newline at end of file + return keeps, removes diff --git a/zfs_autobackup/ThinnerRule.py b/zfs_autobackup/ThinnerRule.py index 0bb1425..ab6f843 100644 --- a/zfs_autobackup/ThinnerRule.py +++ b/zfs_autobackup/ThinnerRule.py @@ -68,4 +68,4 @@ class ThinnerRule: def __str__(self): """get schedule as a schedule string""" - return self.rule_str \ No newline at end of file + return self.rule_str diff --git a/zfs_autobackup/TreeHasher.py b/zfs_autobackup/TreeHasher.py index b787475..0564b12 100644 --- a/zfs_autobackup/TreeHasher.py +++ b/zfs_autobackup/TreeHasher.py @@ -14,7 +14,7 @@ class TreeHasher(): :type block_hasher: BlockHasher """ - self.block_hasher=block_hasher + self.block_hasher = block_hasher def generate(self, start_path): """Use BlockHasher on every file in a tree, yielding the results @@ -28,12 +28,11 @@ class TreeHasher(): for (dirpath, dirnames, filenames) in os.walk(start_path, onerror=walkerror): for f in filenames: - file_path=os.path.join(dirpath, f) + file_path = os.path.join(dirpath, f) if (not os.path.islink(file_path)) and os.path.isfile(file_path): for (chunk_nr, hash) in self.block_hasher.generate(file_path): - yield ( os.path.relpath(file_path,start_path), chunk_nr, hash ) - + yield (os.path.relpath(file_path, start_path), chunk_nr, hash) def compare(self, start_path, generator): """reads from generator and compares blocks @@ -43,18 +42,14 @@ class TreeHasher(): """ - count=0 - - def filter_file_name( file_name, chunk_nr, hexdigest): - return ( chunk_nr, hexdigest ) + count = 0 + def filter_file_name(file_name, chunk_nr, hexdigest): + return (chunk_nr, hexdigest) for file_name, group_generator in itertools.groupby(generator, lambda x: x[0]): - count=count+1 - block_generator=itertools.starmap(filter_file_name, group_generator) - for ( chunk_nr, compare_hexdigest, actual_hexdigest) in self.block_hasher.compare(os.path.join(start_path,file_name), block_generator): - yield ( file_name, chunk_nr, compare_hexdigest, actual_hexdigest ) - - - - + count = count + 1 + block_generator = itertools.starmap(filter_file_name, group_generator) + for (chunk_nr, compare_hexdigest, actual_hexdigest) in self.block_hasher.compare( + os.path.join(start_path, file_name), block_generator): + yield (file_name, chunk_nr, compare_hexdigest, actual_hexdigest) diff --git a/zfs_autobackup/ZfsAuto.py b/zfs_autobackup/ZfsAuto.py index 4027fcb..c8a98fe 100644 --- a/zfs_autobackup/ZfsAuto.py +++ b/zfs_autobackup/ZfsAuto.py @@ -49,13 +49,14 @@ class ZfsAuto(CliBase): self.exclude_paths.append(args.target_path) else: if not args.exclude_received and not args.include_received: - self.verbose("NOTE: Source and target are on the same host, adding --exclude-received to commandline. (use --include-received to overrule)") + self.verbose( + "NOTE: Source and target are on the same host, adding --exclude-received to commandline. (use --include-received to overrule)") args.exclude_received = True if args.test: self.warning("TEST MODE - SIMULATING WITHOUT MAKING ANY CHANGES") - #format all the names + # format all the names self.property_name = args.property_format.format(args.backup_name) self.snapshot_time_format = args.snapshot_format.format(args.backup_name) self.hold_name = args.hold_format.format(args.backup_name) @@ -63,7 +64,8 @@ class ZfsAuto(CliBase): dt = datetime_now(args.utc) self.verbose("") - self.verbose("Current time {} : {}".format(args.utc and "UTC" or " ", dt.strftime("%Y-%m-%d %H:%M:%S"))) + self.verbose( + "Current time {} : {}".format(args.utc and "UTC" or " ", dt.strftime("%Y-%m-%d %H:%M:%S"))) self.verbose("Selecting dataset property : {}".format(self.property_name)) self.verbose("Snapshot format : {}".format(self.snapshot_time_format)) @@ -75,43 +77,40 @@ class ZfsAuto(CliBase): parser = super(ZfsAuto, self).get_parser() - #positional arguments + # positional arguments parser.add_argument('backup_name', metavar='BACKUP-NAME', default=None, nargs='?', help='Name of the backup to select') parser.add_argument('target_path', metavar='TARGET-PATH', default=None, nargs='?', help='Target ZFS filesystem (optional)') - - # SSH options - group=parser.add_argument_group("SSH options") + group = parser.add_argument_group("SSH options") group.add_argument('--ssh-config', metavar='CONFIG-FILE', default=None, help='Custom ssh client config') group.add_argument('--ssh-source', metavar='USER@HOST', default=None, - help='Source host to pull backup from.') + help='Source host to pull backup from.') group.add_argument('--ssh-target', metavar='USER@HOST', default=None, - help='Target host to push backup to.') + help='Target host to push backup to.') - group=parser.add_argument_group("String formatting options") + group = parser.add_argument_group("String formatting options") group.add_argument('--property-format', metavar='FORMAT', default="autobackup:{}", - help='Dataset selection string format. Default: %(default)s') + help='Dataset selection string format. Default: %(default)s') group.add_argument('--snapshot-format', metavar='FORMAT', default="{}-%Y%m%d%H%M%S", - help='ZFS Snapshot string format. Default: %(default)s') + help='ZFS Snapshot string format. Default: %(default)s') group.add_argument('--hold-format', metavar='FORMAT', default="zfs_autobackup:{}", - help='ZFS hold string format. Default: %(default)s') + help='ZFS hold string format. Default: %(default)s') group.add_argument('--strip-path', metavar='N', default=0, type=int, help='Number of directories to strip from target path.') - group=parser.add_argument_group("Selection options") + group = parser.add_argument_group("Selection options") group.add_argument('--ignore-replicated', action='store_true', help=argparse.SUPPRESS) group.add_argument('--exclude-unchanged', metavar='BYTES', default=0, type=int, - help='Exclude datasets that have less than BYTES data changed since any last snapshot. (Use with proxmox HA replication)') + help='Exclude datasets that have less than BYTES data changed since any last snapshot. (Use with proxmox HA replication)') group.add_argument('--exclude-received', action='store_true', - help='Exclude datasets that have the origin of their autobackup: property as "received". ' - 'This can avoid recursive replication between two backup partners.') + help='Exclude datasets that have the origin of their autobackup: property as "received". ' + 'This can avoid recursive replication between two backup partners.') group.add_argument('--include-received', action='store_true', - help=argparse.SUPPRESS) - + help=argparse.SUPPRESS) def regex_argument_type(input_line): """Parses regex arguments into re.Pattern objects""" @@ -119,7 +118,9 @@ class ZfsAuto(CliBase): return re.compile(input_line) except: raise ValueError("Could not parse argument '{}' as a regular expression".format(input_line)) - group.add_argument('--exclude-snapshot-pattern', action='append', default=[], type=regex_argument_type, help="Regular expression to match snapshots that will be ignored.") + + group.add_argument('--exclude-snapshot-pattern', action='append', default=[], type=regex_argument_type, + help="Regular expression to match snapshots that will be ignored.") return parser diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index e3616ce..4eb076f 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -1,4 +1,3 @@ - import argparse from signal import signal, SIGPIPE from .util import output_redir, sigpipe_handler, datetime_now @@ -12,6 +11,7 @@ from .ZfsDataset import ZfsDataset from .ZfsNode import ZfsNode from .ThinnerRule import ThinnerRule + class ZfsAutobackup(ZfsAuto): """The main zfs-autobackup class. Start here, at run() :)""" @@ -73,7 +73,6 @@ class ZfsAutobackup(ZfsAuto): group.add_argument('--no-guid-check', action='store_true', help='Dont check guid of common snapshots. (faster)') - group = parser.add_argument_group("Transfer options") group.add_argument('--no-send', action='store_true', help='Don\'t transfer snapshots (useful for cleanups, or if you want a separate send-cronjob)') @@ -324,8 +323,8 @@ class ZfsAutobackup(ZfsAuto): def make_target_name(self, source_dataset): """make target_name from a source_dataset""" - stripped=source_dataset.lstrip_path(self.args.strip_path) - if stripped!="": + stripped = source_dataset.lstrip_path(self.args.strip_path) + if stripped != "": return self.args.target_path + "/" + stripped else: return self.args.target_path @@ -334,16 +333,20 @@ class ZfsAutobackup(ZfsAuto): """check all target names for collesions etc due to strip-options""" self.debug("Checking target names:") - target_datasets={} + target_datasets = {} for source_dataset in source_datasets: target_name = self.make_target_name(source_dataset) source_dataset.debug("-> {}".format(target_name)) if target_name in target_datasets: - raise Exception("Target collision: Target path {} encountered twice, due to: {} and {}".format(target_name, source_dataset, target_datasets[target_name])) + raise Exception( + "Target collision: Target path {} encountered twice, due to: {} and {}".format(target_name, + source_dataset, + target_datasets[ + target_name])) - target_datasets[target_name]=source_dataset + target_datasets[target_name] = source_dataset # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: def sync_datasets(self, source_node, source_datasets, target_node): @@ -397,7 +400,8 @@ class ZfsAutobackup(ZfsAuto): destroy_incompatible=self.args.destroy_incompatible, send_pipes=send_pipes, recv_pipes=recv_pipes, decrypt=self.args.decrypt, encrypt=self.args.encrypt, - zfs_compressed=self.args.zfs_compressed, force=self.args.force, guid_check=not self.args.no_guid_check) + zfs_compressed=self.args.zfs_compressed, force=self.args.force, + guid_check=not self.args.no_guid_check) except Exception as e: fail_count = fail_count + 1 @@ -406,7 +410,6 @@ class ZfsAutobackup(ZfsAuto): self.verbose("Debug mode, aborting on first error") raise - target_path_dataset = target_node.get_dataset(self.args.target_path) if not self.args.no_thinning: self.thin_missing_targets(target_dataset=target_path_dataset, used_target_datasets=target_datasets) @@ -477,10 +480,10 @@ class ZfsAutobackup(ZfsAuto): ################# select source datasets self.set_title("Selecting") - ( source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, - exclude_received=self.args.exclude_received, - exclude_paths=self.exclude_paths, - exclude_unchanged=self.args.exclude_unchanged) + (source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, + exclude_received=self.args.exclude_received, + exclude_paths=self.exclude_paths, + exclude_unchanged=self.args.exclude_unchanged) if not source_datasets and not excluded_datasets: self.print_error_sources() return 255 @@ -572,7 +575,7 @@ def cli(): signal(SIGPIPE, sigpipe_handler) - failed_datasets=ZfsAutobackup(sys.argv[1:], False).run() + failed_datasets = ZfsAutobackup(sys.argv[1:], False).run() sys.exit(min(failed_datasets, 255)) diff --git a/zfs_autobackup/ZfsAutoverify.py b/zfs_autobackup/ZfsAutoverify.py index 96abd94..906ae0a 100644 --- a/zfs_autobackup/ZfsAutoverify.py +++ b/zfs_autobackup/ZfsAutoverify.py @@ -76,14 +76,14 @@ def verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt, source_snapshot.mount(source_mnt) target_snapshot.mount(target_mnt) - if method=='rsync': + if method == 'rsync': compare_trees_rsync(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt) # elif method == 'tar': # compare_trees_tar(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt) elif method == 'find': compare_trees_find(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt) else: - raise(Exception("program errror, unknown method")) + raise (Exception("program errror, unknown method")) finally: source_snapshot.unmount(source_mnt) @@ -109,7 +109,6 @@ def verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt, # return hashed - # def deacitvate_volume_snapshot(snapshot): # clone_name=get_tmp_clone_name(snapshot) # clone=snapshot.zfs_node.get_dataset(clone_name) @@ -119,13 +118,13 @@ def verify_volume(source_dataset, source_snapshot, target_dataset, target_snapsh """compare the contents of two zfs volume snapshots""" # try: - source_dev= activate_volume_snapshot(source_snapshot) - target_dev= activate_volume_snapshot(target_snapshot) + source_dev = activate_volume_snapshot(source_snapshot) + target_dev = activate_volume_snapshot(target_snapshot) - source_hash= hash_dev(source_snapshot.zfs_node, source_dev) - target_hash= hash_dev(target_snapshot.zfs_node, target_dev) + source_hash = hash_dev(source_snapshot.zfs_node, source_dev) + target_hash = hash_dev(target_snapshot.zfs_node, target_dev) - if source_hash!=target_hash: + if source_hash != target_hash: raise Exception("md5hash difference: {} != {}".format(source_hash, target_hash)) # finally: @@ -150,7 +149,7 @@ class ZfsAutoverify(ZfsAuto): def parse_args(self, argv): """do extra checks on common args""" - args=super(ZfsAutoverify, self).parse_args(argv) + args = super(ZfsAutoverify, self).parse_args(argv) if args.target_path == None: self.log.error("Please specify TARGET-PATH") @@ -161,17 +160,17 @@ class ZfsAutoverify(ZfsAuto): def get_parser(self): """extend common parser with extra stuff needed for zfs-autobackup""" - parser=super(ZfsAutoverify, self).get_parser() + parser = super(ZfsAutoverify, self).get_parser() - group=parser.add_argument_group("Verify options") + group = parser.add_argument_group("Verify options") group.add_argument('--fs-compare', metavar='METHOD', default="find", choices=["find", "rsync"], - help='Compare method to use for filesystems. (find, rsync) Default: %(default)s ') + help='Compare method to use for filesystems. (find, rsync) Default: %(default)s ') return parser def verify_datasets(self, source_mnt, source_datasets, target_node, target_mnt): - fail_count=0 + fail_count = 0 count = 0 for source_dataset in source_datasets: @@ -190,16 +189,17 @@ class ZfsAutoverify(ZfsAuto): target_snapshot = target_dataset.find_snapshot(source_snapshot) if source_snapshot is None or target_snapshot is None: - raise(Exception("Cant find common snapshot")) + raise (Exception("Cant find common snapshot")) target_snapshot.verbose("Verifying...") - if source_dataset.properties['type']=="filesystem": + if source_dataset.properties['type'] == "filesystem": verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt, self.args.fs_compare) - elif source_dataset.properties['type']=="volume": + elif source_dataset.properties['type'] == "volume": verify_volume(source_dataset, source_snapshot, target_dataset, target_snapshot) else: - raise(Exception("{} has unknown type {}".format(source_dataset, source_dataset.properties['type']))) + raise ( + Exception("{} has unknown type {}".format(source_dataset, source_dataset.properties['type']))) except Exception as e: @@ -219,11 +219,10 @@ class ZfsAutoverify(ZfsAuto): def run(self): - source_node=None - source_mnt=None - target_node=None - target_mnt=None - + source_node = None + source_mnt = None + target_node = None + target_mnt = None try: @@ -240,10 +239,10 @@ class ZfsAutoverify(ZfsAuto): ################# select source datasets self.set_title("Selecting") - ( source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, - exclude_received=self.args.exclude_received, - exclude_paths=self.exclude_paths, - exclude_unchanged=self.args.exclude_unchanged) + (source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, + exclude_received=self.args.exclude_received, + exclude_paths=self.exclude_paths, + exclude_unchanged=self.args.exclude_unchanged) if not source_datasets and not excluded_datasets: self.print_error_sources() return 255 @@ -260,7 +259,7 @@ class ZfsAutoverify(ZfsAuto): self.set_title("Verifying") - source_mnt, target_mnt= create_mountpoints(source_node, target_node) + source_mnt, target_mnt = create_mountpoints(source_node, target_node) fail_count = self.verify_datasets( source_mnt=source_mnt, @@ -302,15 +301,13 @@ class ZfsAutoverify(ZfsAuto): cleanup_mountpoint(target_node, target_mnt) - - def cli(): import sys - raise(Exception("This program is incomplete, dont use it yet.")) + raise (Exception("This program is incomplete, dont use it yet.")) signal(SIGPIPE, sigpipe_handler) failed = ZfsAutoverify(sys.argv[1:], False).run() - sys.exit(min(failed,255)) + sys.exit(min(failed, 255)) if __name__ == "__main__": diff --git a/zfs_autobackup/ZfsCheck.py b/zfs_autobackup/ZfsCheck.py index 10fe8ae..61d45fe 100644 --- a/zfs_autobackup/ZfsCheck.py +++ b/zfs_autobackup/ZfsCheck.py @@ -27,14 +27,16 @@ class ZfsCheck(CliBase): parser = super(ZfsCheck, self).get_parser() # positional arguments - parser.add_argument('target', metavar='TARGET', default=None, nargs='?', help='Target to checkum. (can be blockdevice, directory or ZFS snapshot)') + parser.add_argument('target', metavar='TARGET', default=None, nargs='?', + help='Target to checkum. (can be blockdevice, directory or ZFS snapshot)') group = parser.add_argument_group('Checker options') group.add_argument('--block-size', metavar="BYTES", default=4096, help="Read block-size, default %(default)s", type=int) group.add_argument('--count', metavar="COUNT", default=int((100 * (1024 ** 2)) / 4096), - help="Hash chunks of COUNT blocks. Default %(default)s . (CHUNK size is BYTES * COUNT) ", type=int) # 100MiB + help="Hash chunks of COUNT blocks. Default %(default)s . (CHUNK size is BYTES * COUNT) ", + type=int) # 100MiB group.add_argument('--check', '-c', metavar="FILE", default=None, const=True, nargs='?', help="Read hashes from STDIN (or FILE) and compare them") @@ -57,11 +59,10 @@ class ZfsCheck(CliBase): self.verbose("Target : {}".format(args.target)) self.verbose("Block size : {} bytes".format(args.block_size)) self.verbose("Block count : {}".format(args.count)) - self.verbose("Effective chunk size : {} bytes".format(args.count*args.block_size)) - self.verbose("Skip chunk count : {} (checks {:.2f}% of data)".format(args.skip, 100/(1+args.skip))) + self.verbose("Effective chunk size : {} bytes".format(args.count * args.block_size)) + self.verbose("Skip chunk count : {} (checks {:.2f}% of data)".format(args.skip, 100 / (1 + args.skip))) self.verbose("") - return args def prepare_zfs_filesystem(self, snapshot): @@ -106,7 +107,9 @@ class ZfsCheck(CliBase): time.sleep(1) - raise (Exception("Timeout while waiting for /dev entry to appear. (looking in: {}). Hint: did you forget to load the encryption key?".format(locations))) + raise (Exception( + "Timeout while waiting for /dev entry to appear. (looking in: {}). Hint: did you forget to load the encryption key?".format( + locations))) def cleanup_zfs_volume(self, snapshot): """destroys temporary volume snapshot""" @@ -144,34 +147,34 @@ class ZfsCheck(CliBase): """parse input lines and yield items to use in compare functions""" if self.args.check is True: - input_fh=sys.stdin + input_fh = sys.stdin else: - input_fh=open(self.args.check, 'r') + input_fh = open(self.args.check, 'r') last_progress_time = time.time() progress_checked = 0 progress_skipped = 0 - line=input_fh.readline() - skip=0 + line = input_fh.readline() + skip = 0 while line: - i=line.rstrip().split("\t") - #ignores lines without tabs - if (len(i)>1): + i = line.rstrip().split("\t") + # ignores lines without tabs + if (len(i) > 1): - if skip==0: - progress_checked=progress_checked+1 + if skip == 0: + progress_checked = progress_checked + 1 yield i - skip=self.args.skip + skip = self.args.skip else: - skip=skip-1 - progress_skipped=progress_skipped+1 + skip = skip - 1 + progress_skipped = progress_skipped + 1 if self.args.progress and time.time() - last_progress_time > 1: last_progress_time = time.time() self.progress("Checked {} hashes (skipped {})".format(progress_checked, progress_skipped)) - line=input_fh.readline() + line = input_fh.readline() self.verbose("Checked {} hashes (skipped {})".format(progress_checked, progress_skipped)) @@ -224,7 +227,7 @@ class ZfsCheck(CliBase): if "@" in self.args.target: # zfs snapshot - snapshot=self.node.get_dataset(self.args.target) + snapshot = self.node.get_dataset(self.args.target) if not snapshot.exists: raise Exception("ZFS snapshot {} does not exist!".format(snapshot)) dataset_type = snapshot.parent.properties['type'] @@ -240,7 +243,7 @@ class ZfsCheck(CliBase): def cleanup_target(self): if "@" in self.args.target: # zfs snapshot - snapshot=self.node.get_dataset(self.args.target) + snapshot = self.node.get_dataset(self.args.target) if not snapshot.exists: return @@ -253,28 +256,28 @@ class ZfsCheck(CliBase): def run(self): - compare_generator=None - hash_generator=None + compare_generator = None + hash_generator = None try: - prepared_target=self.prepare_target() - is_dir=os.path.isdir(prepared_target) + prepared_target = self.prepare_target() + is_dir = os.path.isdir(prepared_target) - #run as compare + # run as compare if self.args.check is not None: - input_generator=self.generate_input() + input_generator = self.generate_input() if is_dir: compare_generator = self.generate_tree_compare(prepared_target, input_generator) else: - compare_generator=self.generate_file_compare(prepared_target, input_generator) - errors=self.print_errors(compare_generator) - #run as generator + compare_generator = self.generate_file_compare(prepared_target, input_generator) + errors = self.print_errors(compare_generator) + # run as generator else: if is_dir: hash_generator = self.generate_tree_hashes(prepared_target) else: - hash_generator=self.generate_file_hashes(prepared_target) + hash_generator = self.generate_file_hashes(prepared_target) - errors=self.print_hashes(hash_generator) + errors = self.print_hashes(hash_generator) except Exception as e: self.error("Exception: " + str(e)) @@ -286,10 +289,10 @@ class ZfsCheck(CliBase): return 255 finally: - #important to call check_output so that cleanup still functions in case of a broken pipe: + # important to call check_output so that cleanup still functions in case of a broken pipe: # util.check_output() - #close generators, to make sure files are not in use anymore when cleaning up + # close generators, to make sure files are not in use anymore when cleaning up if hash_generator is not None: hash_generator.close() if compare_generator is not None: @@ -302,8 +305,8 @@ class ZfsCheck(CliBase): def cli(): import sys signal(SIGPIPE, sigpipe_handler) - failed=ZfsCheck(sys.argv[1:], False).run() - sys.exit(min(failed,255)) + failed = ZfsCheck(sys.argv[1:], False).run() + sys.exit(min(failed, 255)) if __name__ == "__main__": diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 9efa79d..44cd6c5 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1,4 +1,3 @@ - import re from datetime import datetime import sys @@ -29,28 +28,27 @@ class ZfsDataset: self.zfs_node = zfs_node self.name = name # full name - #caching - self.__snapshots=None #type: None|list[ZfsDataset] - self.__written_since_ours=None #type: None|int - self.__exists_check=None #type: None|bool - self.__properties=None #type: None|dict[str,str] - self.__recursive_datasets=None #type: None|list[ZfsDataset] - self.__datasets=None #type: None|list[ZfsDataset] + # caching + self.__snapshots = None # type: None|list[ZfsDataset] + self.__written_since_ours = None # type: None|int + self.__exists_check = None # type: None|bool + self.__properties = None # type: None|dict[str,str] + self.__recursive_datasets = None # type: None|list[ZfsDataset] + self.__datasets = None # type: None|list[ZfsDataset] self.invalidate_cache() self.force_exists = force_exists - def invalidate_cache(self): """clear caches""" # CachedProperty.clear(self) self.force_exists = None - self.__snapshots=None - self.__written_since_ours=None - self.__exists_check=None - self.__properties=None - self.__recursive_datasets=None - self.__datasets=None + self.__snapshots = None + self.__written_since_ours = None + self.__exists_check = None + self.__properties = None + self.__recursive_datasets = None + self.__datasets = None def __repr__(self): return "{}: {}".format(self.zfs_node, self.name) @@ -93,7 +91,6 @@ class ZfsDataset: """ self.zfs_node.debug("{}: {}".format(self.name, txt)) - def split_path(self): """return the path elements as an array""" return self.name.split("/") @@ -147,14 +144,11 @@ class ZfsDataset: if not self.is_snapshot: return False - for pattern in self.zfs_node.exclude_snapshot_patterns: if pattern.search(self.name) is not None: self.debug("Excluded (path matches snapshot exclude pattern)") return True - - def is_selected(self, value, source, inherited, exclude_received, exclude_paths, exclude_unchanged): """determine if dataset should be selected for backup (called from ZfsNode) @@ -290,9 +284,9 @@ class ZfsDataset: if self.__exists_check is None: self.debug("Checking if dataset exists") - self.__exists_check=(len(self.zfs_node.run(tab_split=True, cmd=["zfs", "list", self.name], readonly=True, - valid_exitcodes=[0, 1], - hide_errors=True)) > 0) + self.__exists_check = (len(self.zfs_node.run(tab_split=True, cmd=["zfs", "list", self.name], readonly=True, + valid_exitcodes=[0, 1], + hide_errors=True)) > 0) return self.__exists_check @@ -454,7 +448,6 @@ class ZfsDataset: seconds = time.mktime(dt.timetuple()) return seconds - # def add_virtual_snapshot(self, snapshot): # """pretend a snapshot exists (usefull in test mode)""" # @@ -474,18 +467,15 @@ class ZfsDataset: :rtype: list[ZfsDataset] """ - #cached? + # cached? if self.__snapshots is None: - - self.debug("Getting snapshots") cmd = [ "zfs", "list", "-d", "1", "-r", "-t", "snapshot", "-H", "-o", "name", self.name ] - self.__snapshots=self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) - + self.__snapshots = self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) return self.__snapshots @@ -517,7 +507,7 @@ class ZfsDataset: """ for snapshot in snapshots: - if snapshot.snapshot_name==self.snapshot_name: + if snapshot.snapshot_name == self.snapshot_name: return snapshot return None @@ -571,7 +561,6 @@ class ZfsDataset: """get number of bytes written since our last snapshot""" if self.__written_since_ours is None: - latest_snapshot = self.our_snapshots[-1] self.debug("Getting bytes written since our last snapshot") @@ -579,7 +568,7 @@ class ZfsDataset: output = self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0]) - self.__written_since_ours=int(output[0]) + self.__written_since_ours = int(output[0]) return self.__written_since_ours @@ -612,14 +601,13 @@ class ZfsDataset: """ if self.__recursive_datasets is None: - self.debug("Getting all recursive datasets under us") names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name ]) - self.__recursive_datasets=self.zfs_node.get_datasets(names[1:], force_exists=True) + self.__recursive_datasets = self.zfs_node.get_datasets(names[1:], force_exists=True) return self.__recursive_datasets @@ -632,14 +620,13 @@ class ZfsDataset: """ if self.__datasets is None: - self.debug("Getting all datasets under us") names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ "zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name ]) - self.__datasets=self.zfs_node.get_datasets(names[1:], force_exists=True) + self.__datasets = self.zfs_node.get_datasets(names[1:], force_exists=True) return self.__datasets @@ -804,7 +791,7 @@ class ZfsDataset: if self.properties['encryption'] != 'off' and self.properties['keystatus'] == 'unavailable': return - self.zfs_node.run(["zfs", "mount", self.name], valid_exitcodes=[0,1]) + self.zfs_node.run(["zfs", "mount", self.name], valid_exitcodes=[0, 1]) def transfer_snapshot(self, target_snapshot, features, prev_snapshot, show_progress, filter_properties, set_properties, ignore_recv_exit_code, resume_token, @@ -960,7 +947,6 @@ class ZfsDataset: # target_dataset.error("Cant find common snapshot with source.") raise (Exception("Cant find common snapshot with target.")) - def find_incompatible_snapshots(self, common_snapshot, raw): """returns a list[snapshots] that is incompatible for a zfs recv onto the common_snapshot. all direct followup snapshots with written=0 are @@ -1006,7 +992,6 @@ class ZfsDataset: return allowed_filter_properties, allowed_set_properties - def _pre_clean(self, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): """cleanup old stuff before starting snapshot syncing @@ -1021,7 +1006,7 @@ class ZfsDataset: # on source: delete all obsoletes that are not in target_transfers (except common snapshot) for source_snapshot in self.snapshots: if (source_snapshot in source_obsoletes - and source_common_snapshot!=source_snapshot + and source_common_snapshot != source_snapshot and source_snapshot.find_snapshot_in_list(target_transfers) is None): source_snapshot.destroy() @@ -1029,7 +1014,8 @@ class ZfsDataset: if target_dataset.exists: for target_snapshot in target_dataset.snapshots: if (target_snapshot in target_obsoletes) \ - and (not source_common_snapshot or (target_snapshot.snapshot_name != source_common_snapshot.snapshot_name)): + and (not source_common_snapshot or ( + target_snapshot.snapshot_name != source_common_snapshot.snapshot_name)): if target_snapshot.exists: target_snapshot.destroy() @@ -1089,36 +1075,40 @@ class ZfsDataset: # start with snapshots that already exist, minus imcompatibles if target_dataset.exists: - possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots if snapshot not in incompatible_target_snapshots] + possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots if + snapshot not in incompatible_target_snapshots] else: possible_target_snapshots = [] # add all snapshots from the source, starting after the common snapshot if it exists if source_common_snapshot: - source_snapshot=self.find_next_snapshot(source_common_snapshot ) + source_snapshot = self.find_next_snapshot(source_common_snapshot) else: if self.snapshots: - source_snapshot=self.snapshots[0] + source_snapshot = self.snapshots[0] else: - source_snapshot=None + source_snapshot = None while source_snapshot: # we want it? if (also_other_snapshots or source_snapshot.is_ours()) and not source_snapshot.is_excluded: # create virtual target snapshot - target_snapshot=target_dataset.zfs_node.get_dataset(target_dataset.filesystem_name + "@" + source_snapshot.snapshot_name, force_exists=False) + target_snapshot = target_dataset.zfs_node.get_dataset( + target_dataset.filesystem_name + "@" + source_snapshot.snapshot_name, force_exists=False) possible_target_snapshots.append(target_snapshot) source_snapshot = self.find_next_snapshot(source_snapshot) ### 3: Let the thinner decide what it wants by looking at all the possible target_snaphots at once if possible_target_snapshots: - (target_keeps, target_obsoletes)=target_dataset.zfs_node.thin_list(possible_target_snapshots, keep_snapshots=[possible_target_snapshots[-1]]) + (target_keeps, target_obsoletes) = target_dataset.zfs_node.thin_list(possible_target_snapshots, + keep_snapshots=[ + possible_target_snapshots[-1]]) else: target_keeps = [] target_obsoletes = [] ### 4: Look at what the thinner wants and create a list of snapshots we still need to transfer - target_transfers=[] + target_transfers = [] for target_keep in target_keeps: if not target_keep.exists: target_transfers.append(target_keep) @@ -1188,7 +1178,7 @@ class ZfsDataset: # keep data encrypted by sending it raw (including properties) raw = True - (source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, + (source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots) = \ self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, guid_check=guid_check, raw=raw) @@ -1203,7 +1193,7 @@ class ZfsDataset: target_dataset.handle_incompatible_snapshots(incompatible_target_snapshots, destroy_incompatible) # now actually transfer the snapshots, if we want - if no_send or len(target_transfers)==0: + if no_send or len(target_transfers) == 0: return # check if we can resume @@ -1221,11 +1211,11 @@ class ZfsDataset: # now actually transfer the snapshots do_rollback = rollback - prev_source_snapshot=source_common_snapshot - prev_target_snapshot=target_dataset.find_snapshot(source_common_snapshot) + prev_source_snapshot = source_common_snapshot + prev_target_snapshot = target_dataset.find_snapshot(source_common_snapshot) for target_snapshot in target_transfers: - source_snapshot=self.find_snapshot(target_snapshot) + source_snapshot = self.find_snapshot(target_snapshot) # do the rollback, one time at first transfer if do_rollback: @@ -1312,8 +1302,8 @@ class ZfsDataset: self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - #invalidate cache - self.__properties=None + # invalidate cache + self.__properties = None def inherit(self, prop): """inherit zfs property""" @@ -1326,5 +1316,5 @@ class ZfsDataset: self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - #invalidate cache - self.__properties=None + # invalidate cache + self.__properties = None diff --git a/zfs_autobackup/ZfsPool.py b/zfs_autobackup/ZfsPool.py index 11976ab..53a3c57 100644 --- a/zfs_autobackup/ZfsPool.py +++ b/zfs_autobackup/ZfsPool.py @@ -60,4 +60,4 @@ class ZfsPool(): if value == 'enabled' or value == 'active': ret.append(feature) - return ret \ No newline at end of file + return ret diff --git a/zfs_autobackup/compressors.py b/zfs_autobackup/compressors.py index 40b39e5..b1ce8ec 100644 --- a/zfs_autobackup/compressors.py +++ b/zfs_autobackup/compressors.py @@ -7,69 +7,72 @@ COMPRESS_CMDS = { 'gzip': { 'cmd': 'gzip', - 'args': [ '-3' ], + 'args': ['-3'], 'dcmd': 'zcat', 'dargs': [], }, 'pigz-fast': { 'cmd': 'pigz', - 'args': [ '-3' ], + 'args': ['-3'], 'dcmd': 'pigz', - 'dargs': [ '-dc' ], + 'dargs': ['-dc'], }, 'pigz-slow': { 'cmd': 'pigz', - 'args': [ '-9' ], + 'args': ['-9'], 'dcmd': 'pigz', - 'dargs': [ '-dc' ], + 'dargs': ['-dc'], }, 'zstd-fast': { 'cmd': 'zstdmt', - 'args': [ '-3' ], + 'args': ['-3'], 'dcmd': 'zstdmt', - 'dargs': [ '-dc' ], + 'dargs': ['-dc'], }, 'zstd-slow': { 'cmd': 'zstdmt', - 'args': [ '-19' ], + 'args': ['-19'], 'dcmd': 'zstdmt', - 'dargs': [ '-dc' ], + 'dargs': ['-dc'], }, 'zstd-adapt': { 'cmd': 'zstdmt', - 'args': [ '--adapt' ], + 'args': ['--adapt'], 'dcmd': 'zstdmt', - 'dargs': [ '-dc' ], + 'dargs': ['-dc'], }, 'xz': { 'cmd': 'xz', 'args': [], 'dcmd': 'xz', - 'dargs': [ '-d' ], + 'dargs': ['-d'], }, 'lzo': { 'cmd': 'lzop', 'args': [], 'dcmd': 'lzop', - 'dargs': [ '-dfc' ], + 'dargs': ['-dfc'], }, 'lz4': { 'cmd': 'lz4', 'args': [], 'dcmd': 'lz4', - 'dargs': [ '-dc' ], + 'dargs': ['-dc'], }, } + def compress_cmd(compressor): - ret=[ COMPRESS_CMDS[compressor]['cmd'] ] - ret.extend( COMPRESS_CMDS[compressor]['args']) + ret = [COMPRESS_CMDS[compressor]['cmd']] + ret.extend(COMPRESS_CMDS[compressor]['args']) return ret + def decompress_cmd(compressor): - ret= [ COMPRESS_CMDS[compressor]['dcmd'] ] + ret = [COMPRESS_CMDS[compressor]['dcmd']] ret.extend(COMPRESS_CMDS[compressor]['dargs']) return ret + def choices(): return COMPRESS_CMDS.keys() diff --git a/zfs_autobackup/util.py b/zfs_autobackup/util.py index db6f07c..76d75ab 100644 --- a/zfs_autobackup/util.py +++ b/zfs_autobackup/util.py @@ -1,4 +1,3 @@ - # NOTE: surprisingly sha1 in via python3 is faster than the native sha1sum utility, even in the way we use below! import os import platform @@ -9,19 +8,18 @@ from datetime import datetime def tmp_name(suffix=""): """create temporary name unique to this process and node. always retruns the same result during the same execution""" - #we could use uuids but those are ugly and confusing - name="{}-{}-{}".format( - os.path.basename(sys.argv[0]).replace(" ","_"), + # we could use uuids but those are ugly and confusing + name = "{}-{}-{}".format( + os.path.basename(sys.argv[0]).replace(" ", "_"), platform.node(), os.getpid()) - name=name+suffix + name = name + suffix return name def get_tmp_clone_name(snapshot): - pool=snapshot.zfs_node.get_pool(snapshot) - return pool.name+"/"+tmp_name() - + pool = snapshot.zfs_node.get_pool(snapshot) + return pool.name + "/" + tmp_name() def output_redir(): @@ -33,10 +31,12 @@ def output_redir(): os.dup2(devnull, sys.stdout.fileno()) os.dup2(devnull, sys.stderr.fileno()) + def sigpipe_handler(sig, stack): - #redir output so we dont get more SIGPIPES during cleanup. (which my try to write to stdout) + # redir output so we dont get more SIGPIPES during cleanup. (which my try to write to stdout) output_redir() - #deb('redir') + # deb('redir') + # def check_output(): # """make sure stdout still functions. if its broken, this will trigger a SIGPIPE which will be handled by the sigpipe_handler.""" @@ -55,9 +55,11 @@ def sigpipe_handler(sig, stack): # This function will be mocked during unit testing. -datetime_now_mock=None +datetime_now_mock = None + + def datetime_now(utc): if datetime_now_mock is None: - return( datetime.utcnow() if utc else datetime.now()) + return (datetime.utcnow() if utc else datetime.now()) else: return datetime_now_mock From b54c10ef9a453fdb8f4889426cefaa912bbd58c6 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 24 Sep 2024 18:29:45 +0200 Subject: [PATCH 015/135] got rid of the rest of CachedProperty. (its too hacky, and confuses pycharm as well, and now we can actually partually invalidate caches. take that past self!) --- zfs_autobackup/CachedProperty.py | 39 ------------------------ zfs_autobackup/ZfsDataset.py | 14 --------- zfs_autobackup/ZfsNode.py | 52 +++++++++++++++++--------------- zfs_autobackup/ZfsPool.py | 27 +++++++++-------- 4 files changed, 43 insertions(+), 89 deletions(-) delete mode 100644 zfs_autobackup/CachedProperty.py diff --git a/zfs_autobackup/CachedProperty.py b/zfs_autobackup/CachedProperty.py deleted file mode 100644 index 1f25e50..0000000 --- a/zfs_autobackup/CachedProperty.py +++ /dev/null @@ -1,39 +0,0 @@ -# NOTE: this should inherit from (object) to function correctly with python 2.7 -class CachedProperty(object): - """ A property that is only computed once per instance and - then stores the result in _cached_properties of the object. - - Source: https://github.com/bottlepy/bottle/commit/fa7733e075da0d790d809aa3d2f53071897e6f76 - """ - - def __init__(self, func): - self.__doc__ = getattr(func, '__doc__') - self.func = func - - def __get__(self, obj, cls): - if obj is None: - return self - - propname = self.func.__name__ - - if not hasattr(obj, '_cached_properties'): - obj._cached_properties = {} - - if propname not in obj._cached_properties: - obj._cached_properties[propname] = self.func(obj) - # value = obj.__dict__[propname] = self.func(obj) - - return obj._cached_properties[propname] - - @staticmethod - def clear(obj): - """clears cache of obj""" - if hasattr(obj, '_cached_properties'): - obj._cached_properties = {} - - @staticmethod - def is_cached(obj, propname): - if hasattr(obj, '_cached_properties') and propname in obj._cached_properties: - return True - else: - return False diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 44cd6c5..055e885 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -41,7 +41,6 @@ class ZfsDataset: def invalidate_cache(self): """clear caches""" - # CachedProperty.clear(self) self.force_exists = None self.__snapshots = None self.__written_since_ours = None @@ -448,19 +447,6 @@ class ZfsDataset: seconds = time.mktime(dt.timetuple()) return seconds - # def add_virtual_snapshot(self, snapshot): - # """pretend a snapshot exists (usefull in test mode)""" - # - # # NOTE: we could just call self.snapshots.append() but this would trigger a zfs list which is not always needed. - # if CachedProperty.is_cached(self, 'snapshots'): - # # already cached so add it - # print ("ADDED") - # self.snapshots.append(snapshot) - # else: - # # self.snapshots will add it when requested - # print ("ADDED VIRT") - # self._virtual_snapshots.append(snapshot) - @property def snapshots(self): """get all snapshots of this dataset diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index b7640e9..cd59b19 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -8,7 +8,6 @@ import time from .ExecuteNode import ExecuteNode from .Thinner import Thinner -from .CachedProperty import CachedProperty from .ZfsPool import ZfsPool from .ZfsDataset import ZfsDataset from .ExecuteNode import ExecuteError @@ -18,7 +17,8 @@ from .util import datetime_now class ZfsNode(ExecuteNode): """a node that contains zfs datasets. implements global (systemwide/pool wide) zfs commands""" - def __init__(self, logger, utc=False, snapshot_time_format="", hold_name="", ssh_config=None, ssh_to=None, readonly=False, + def __init__(self, logger, utc=False, snapshot_time_format="", hold_name="", ssh_config=None, ssh_to=None, + readonly=False, description="", debug_output=False, thinner=None, exclude_snapshot_patterns=[]): @@ -30,6 +30,9 @@ class ZfsNode(ExecuteNode): self.logger = logger + self.__supported_send_options = None + self.__supported_recv_options = None + self.exclude_snapshot_patterns = exclude_snapshot_patterns if ssh_config: @@ -74,27 +77,30 @@ class ZfsNode(ExecuteNode): else: return (keep_snapshots, []) - @CachedProperty + @property def supported_send_options(self): """list of supported options, for optimizing sends""" # not every zfs implementation supports them all - ret = [] - for option in ["-L", "-e", "-c"]: - if self.valid_command(["zfs", "send", option, "zfs_autobackup_option_test"]): - ret.append(option) - return ret + if self.__supported_send_options is None: + self.__supported_send_options = [] + for option in ["-L", "-e", "-c"]: + if self.valid_command(["zfs", "send", option, "zfs_autobackup_option_test"]): + self.__supported_send_options.append(option) + return self.__supported_send_options - @CachedProperty + @property def supported_recv_options(self): """list of supported options""" # not every zfs implementation supports them all - ret = [] - for option in ["-s"]: - if self.valid_command(["zfs", "recv", option, "zfs_autobackup_option_test"]): - ret.append(option) - return ret + if self.__supported_recv_options is None: + self.__supported_recv_options = [] + for option in ["-s"]: + if self.valid_command(["zfs", "recv", option, "zfs_autobackup_option_test"]): + self.__supported_recv_options.append(option) + + return self.__supported_recv_options def valid_command(self, cmd): """test if a specified zfs options are valid exit code. use this to determine support options""" @@ -129,13 +135,12 @@ class ZfsNode(ExecuteNode): """ - ret=[] + ret = [] for name in names: ret.append(self.get_dataset(name, force_exists)) return ret - # def reset_progress(self): # """reset progress output counters""" # self._progress_total_bytes = 0 @@ -275,7 +280,6 @@ class ZfsNode(ExecuteNode): property_name ]) - # The returnlist of selected ZfsDataset's: selected_filesystems = [] excluded_filesystems = [] @@ -298,14 +302,14 @@ class ZfsNode(ExecuteNode): source = raw_source # determine it - selected=dataset.is_selected(value=value, source=source, inherited=inherited, exclude_received=exclude_received, - exclude_paths=exclude_paths, exclude_unchanged=exclude_unchanged) + selected = dataset.is_selected(value=value, source=source, inherited=inherited, + exclude_received=exclude_received, + exclude_paths=exclude_paths, exclude_unchanged=exclude_unchanged) - if selected==True: + if selected == True: selected_filesystems.append(dataset) - elif selected==False: + elif selected == False: excluded_filesystems.append(dataset) - #returns None when no property is set. + # returns None when no property is set. - - return ( selected_filesystems, excluded_filesystems) + return (selected_filesystems, excluded_filesystems) diff --git a/zfs_autobackup/ZfsPool.py b/zfs_autobackup/ZfsPool.py index 53a3c57..f0cdf2e 100644 --- a/zfs_autobackup/ZfsPool.py +++ b/zfs_autobackup/ZfsPool.py @@ -1,6 +1,3 @@ -from .CachedProperty import CachedProperty - - class ZfsPool(): """a zfs pool""" @@ -10,6 +7,10 @@ class ZfsPool(): self.zfs_node = zfs_node self.name = name + self.__properties = None + + def invalidate_cache(self): + self.__properties = None def __repr__(self): return "{}: {}".format(self.zfs_node, self.name) @@ -32,22 +33,24 @@ class ZfsPool(): def debug(self, txt): self.zfs_node.debug("zpool {}: {}".format(self.name, txt)) - @CachedProperty + @property def properties(self): """all zpool properties""" - self.debug("Getting zpool properties") + if self.__properties is None: - cmd = [ - "zpool", "get", "-H", "-p", "all", self.name - ] + self.debug("Getting zpool properties") - ret = {} + cmd = [ + "zpool", "get", "-H", "-p", "all", self.name + ] - for pair in self.zfs_node.run(tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]): - ret[pair[1]] = pair[2] + self.__properties = {} - return ret + for pair in self.zfs_node.run(tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]): + self.__properties[pair[1]] = pair[2] + + return self.__properties @property def features(self): From 85819a96d5921e1cb82b47c923dbce5d39772e6e Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Sep 2024 19:04:37 +0200 Subject: [PATCH 016/135] fix #257 --- zfs_autobackup/ZfsAutobackup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 4eb076f..ff2387e 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -47,6 +47,9 @@ class ZfsAutobackup(ZfsAuto): if args.compress and args.zfs_compressed: self.warning("Using --compress with --zfs-compressed, might be inefficient.") + if args.decrypt: + self.warning("Properties will not be sent over for datasets that will be decrypted. (zfs bug https://github.com/openzfs/zfs/issues/16275)") + return args def get_parser(self): From c9cbe424d7fee11abde79c1c0342adc17a0eb9ae Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Sep 2024 20:29:29 +0200 Subject: [PATCH 017/135] major change: we now always filter all properties that start with "autobackup:". fixes #150 and #221 --- tests/test_zfsautobackup31.py | 29 +++++++++++++++++++++++++---- zfs_autobackup/ZfsAuto.py | 19 +++++++------------ zfs_autobackup/ZfsAutobackup.py | 4 ++++ zfs_autobackup/ZfsDataset.py | 5 +++++ 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/tests/test_zfsautobackup31.py b/tests/test_zfsautobackup31.py index 17de12f..e8650a7 100644 --- a/tests/test_zfsautobackup31.py +++ b/tests/test_zfsautobackup31.py @@ -55,23 +55,44 @@ test_target1/test_source2/fs2/sub@test-20101111000001 shelltest("zfs create test_target1/b") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1/a --no-progress --verbose --debug".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1/a --no-progress --verbose --debug --allow-empty".split(" ")).run()) + + #NOTE: since v3.4 this changed. autobackup: properties are filtered. So its up to the admin to reset this property on the other side: + shelltest("zfs set autobackup:test=true test_target1/a") with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1/b --no-progress --verbose".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1/b --no-progress --verbose --allow-empty".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t snapshot test_target1") #NOTE: it wont backup test_target1/a/test_source2/fs2/sub to test_target1/b since it doesnt have the zfs_autobackup property anymore. - self.assertMultiLineEqual(r,""" + self.assertMultiLineEqual(""" +test_target1/a@test-20101111000001 +test_target1/a/test_source1@test-20101111000001 test_target1/a/test_source1/fs1@test-20101111000000 +test_target1/a/test_source1/fs1@test-20101111000001 test_target1/a/test_source1/fs1/sub@test-20101111000000 +test_target1/a/test_source1/fs1/sub@test-20101111000001 +test_target1/a/test_source2@test-20101111000001 +test_target1/a/test_source2/fs2@test-20101111000001 test_target1/a/test_source2/fs2/sub@test-20101111000000 +test_target1/a/test_source2/fs2/sub@test-20101111000001 test_target1/b/test_source1/fs1@test-20101111000000 +test_target1/b/test_source1/fs1@test-20101111000001 test_target1/b/test_source1/fs1/sub@test-20101111000000 +test_target1/b/test_source1/fs1/sub@test-20101111000001 test_target1/b/test_source2/fs2/sub@test-20101111000000 +test_target1/b/test_source2/fs2/sub@test-20101111000001 +test_target1/b/test_target1/a@test-20101111000001 +test_target1/b/test_target1/a/test_source1@test-20101111000001 test_target1/b/test_target1/a/test_source1/fs1@test-20101111000000 +test_target1/b/test_target1/a/test_source1/fs1@test-20101111000001 test_target1/b/test_target1/a/test_source1/fs1/sub@test-20101111000000 -""") +test_target1/b/test_target1/a/test_source1/fs1/sub@test-20101111000001 +test_target1/b/test_target1/a/test_source2@test-20101111000001 +test_target1/b/test_target1/a/test_source2/fs2@test-20101111000001 +test_target1/b/test_target1/a/test_source2/fs2/sub@test-20101111000000 +test_target1/b/test_target1/a/test_source2/fs2/sub@test-20101111000001 +""",r) def test_zfs_compressed(self): diff --git a/zfs_autobackup/ZfsAuto.py b/zfs_autobackup/ZfsAuto.py index c8a98fe..58ecc66 100644 --- a/zfs_autobackup/ZfsAuto.py +++ b/zfs_autobackup/ZfsAuto.py @@ -41,17 +41,14 @@ class ZfsAuto(CliBase): # may still need to be used to explicitly exclude a backup with the 'received' source property to avoid accidental # recursive replication of a zvol that is currently being received in another session (as it will have changes). + # Follow up note: This isnt a problem anymore since v3.4, we now filter the autobackup-property by default to prevent these difficult issues. + self.exclude_paths = [] if args.ssh_source == args.ssh_target: if args.target_path: # target and source are the same, make sure to exclude target_path self.verbose("NOTE: Source and target are on the same host, excluding target-path from selection.") self.exclude_paths.append(args.target_path) - else: - if not args.exclude_received and not args.include_received: - self.verbose( - "NOTE: Source and target are on the same host, adding --exclude-received to commandline. (use --include-received to overrule)") - args.exclude_received = True if args.test: self.warning("TEST MODE - SIMULATING WITHOUT MAKING ANY CHANGES") @@ -107,10 +104,9 @@ class ZfsAuto(CliBase): group.add_argument('--exclude-unchanged', metavar='BYTES', default=0, type=int, help='Exclude datasets that have less than BYTES data changed since any last snapshot. (Use with proxmox HA replication)') group.add_argument('--exclude-received', action='store_true', - help='Exclude datasets that have the origin of their autobackup: property as "received". ' - 'This can avoid recursive replication between two backup partners.') - group.add_argument('--include-received', action='store_true', - help=argparse.SUPPRESS) + help='Exclude datasets that have the origin of their autobackup: property as "received".' , ) + # group.add_argument('--include-received', action='store_true', + # help=argparse.SUPPRESS) def regex_argument_type(input_line): """Parses regex arguments into re.Pattern objects""" @@ -126,9 +122,8 @@ class ZfsAuto(CliBase): def print_error_sources(self): self.error( - "No source filesystems selected, please do a 'zfs set autobackup:{0}=true' on the source datasets " - "you want to select.".format( - self.args.backup_name)) + "No source filesystems selected, please do a 'zfs set {}=true' on the source datasets " + "you want to select.".format(self.property_name)) def make_target_name(self, source_dataset): """make target_name from a source_dataset""" diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index ff2387e..51e04c2 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -50,6 +50,8 @@ class ZfsAutobackup(ZfsAuto): if args.decrypt: self.warning("Properties will not be sent over for datasets that will be decrypted. (zfs bug https://github.com/openzfs/zfs/issues/16275)") + + return args def get_parser(self): @@ -93,6 +95,7 @@ class ZfsAutobackup(ZfsAuto): group.add_argument('--set-properties', metavar='PROPERTY=VALUE,...', type=str, help='List of propererties to override when receiving filesystems. (you can still restore ' 'them with zfs inherit -S)') + group.add_argument('--rollback', action='store_true', help='Rollback changes to the latest target snapshot before starting. (normally you can ' 'prevent changes by setting the readonly property on the target_path to on)') @@ -439,6 +442,7 @@ class ZfsAutobackup(ZfsAuto): if self.args.clear_refreservation: filter_properties.append("refreservation") + return filter_properties def set_properties_list(self): diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 055e885..58f6341 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1188,6 +1188,11 @@ class ZfsDataset: (active_filter_properties, active_set_properties) = self.get_allowed_properties(filter_properties, set_properties) + # always filter properties that start with 'autobackup:' (https://github.com/psy0rz/zfs_autobackup/issues/221) + for property in self.properties: + if property.startswith('autobackup:'): + active_filter_properties.append(property) + # encrypt at target? if encrypt and not raw: # filter out encryption properties to let encryption on the target take place From 248ba043182ede503c12022443db1caf2aed65c4 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Sep 2024 23:56:06 +0200 Subject: [PATCH 018/135] fix test mode snapshot caching --- zfs_autobackup/ZfsDataset.py | 9 ++++++--- zfs_autobackup/ZfsNode.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 58f6341..ac18f5a 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -465,13 +465,16 @@ class ZfsDataset: return self.__snapshots - def cache_snapshot(self, snapshot): - """Update our snapshot cache (if we have any) + def cache_snapshot(self, snapshot, force=False): + """Update our snapshot cache (if we have any). Use force if you want to force the caching, potentially triggering a zfs list Args: :type snapshot: ZfsDataset """ - if self.__snapshots is not None: + if force: + self.snapshots.append(snapshot) + + elif self.__snapshots is not None: self.__snapshots.append(snapshot) @property diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index cd59b19..70a89b7 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -236,7 +236,7 @@ class ZfsNode(ExecuteNode): pools[pool].append(snapshot) - dataset.cache_snapshot(snapshot) + dataset.cache_snapshot(snapshot, force=self.readonly) if not pools: self.verbose("No changes anywhere: not creating snapshots.") From 39026e5ff62110373ce718f0303904d22d8864be Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 25 Sep 2024 00:08:57 +0200 Subject: [PATCH 019/135] wip: bookmarks --- zfs_autobackup/ZfsDataset.py | 92 ++++++++++++++++++++++++++---------- zfs_autobackup/ZfsNode.py | 2 +- 2 files changed, 67 insertions(+), 27 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index ac18f5a..ac74125 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -35,8 +35,8 @@ class ZfsDataset: self.__properties = None # type: None|dict[str,str] self.__recursive_datasets = None # type: None|list[ZfsDataset] self.__datasets = None # type: None|list[ZfsDataset] + self.__bookmarks = None # type: None|list[ZfsDataset] - self.invalidate_cache() self.force_exists = force_exists def invalidate_cache(self): @@ -48,6 +48,7 @@ class ZfsDataset: self.__properties = None self.__recursive_datasets = None self.__datasets = None + self.__bookmarks = None def __repr__(self): return "{}: {}".format(self.zfs_node, self.name) @@ -124,19 +125,28 @@ class ZfsDataset: return self.name @property - def snapshot_name(self): - """snapshot part of the name""" - if not self.is_snapshot: - raise (Exception("This is not a snapshot")) + def suffix(self): + """snapshot or bookmark part of the name""" + if self.is_snapshot: + (filesystem, snapshot_name) = self.name.split("@") + return snapshot_name + elif self.is_bookmark: + (filesystem, bookmark_name) = self.name.split("#") + return bookmark_name - (filesystem, snapshot_name) = self.name.split("@") - return snapshot_name + raise (Exception("This is not a snapshot or bookmark")) @property def is_snapshot(self): """true if this dataset is a snapshot""" return self.name.find("@") != -1 + @property + def is_bookmark(self): + """true if this dataset is a bookmark""" + return self.name.find("#") != -1 + + @property def is_excluded(self): """true if this dataset is a snapshot and matches the exclude pattern""" @@ -333,7 +343,7 @@ class ZfsDataset: :type fail_exception: bool """ - if verbose: + if verbose and not self.is_bookmark: self.verbose("Destroying") else: self.debug("Destroying") @@ -427,7 +437,7 @@ class ZfsDataset: """ try: - dt = datetime.strptime(self.snapshot_name, self.zfs_node.snapshot_time_format) + dt = datetime.strptime(self.suffix, self.zfs_node.snapshot_time_format) except ValueError: return None @@ -496,7 +506,7 @@ class ZfsDataset: """ for snapshot in snapshots: - if snapshot.snapshot_name == self.snapshot_name: + if snapshot.suffix == self.suffix: return snapshot return None @@ -516,10 +526,10 @@ class ZfsDataset: if not isinstance(snapshot, ZfsDataset): snapshot_name = snapshot else: - snapshot_name = snapshot.snapshot_name + snapshot_name = snapshot.suffix for snapshot in self.snapshots: - if snapshot.snapshot_name == snapshot_name: + if snapshot.suffix == snapshot_name: return snapshot return None @@ -535,11 +545,11 @@ class ZfsDataset: if not isinstance(snapshot, ZfsDataset): snapshot_name = snapshot else: - snapshot_name = snapshot.snapshot_name + snapshot_name = snapshot.suffix index = 0 for snapshot in self.snapshots: - if snapshot.snapshot_name == snapshot_name: + if snapshot.suffix == snapshot_name: return index index = index + 1 @@ -580,6 +590,37 @@ class ZfsDataset: return True + def bookmark(self): + """Bookmark this snapshot""" + + if not self.is_snapshot: + raise(Exception("Can only bookmark a snapshot!")) + + self.debug("Bookmarking") + + cmd = [ + "zfs", "bookmark", self.name, "#"+self.suffix + ] + + self.zfs_node.run(cmd=cmd) + + + @property + def bookmarks(self): + + if self.__bookmarks is None: + self.debug("Getting bookmarks") + + cmd = [ + "zfs", "list", "-d", "1", "-r", "-t", "bookmark", "-H", "-o", "name", self.name + ] + + self.__bookmarks = self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) + + + return self.__bookmarks + + @property def recursive_datasets(self, types="filesystem,volume"): """get all (non-snapshot) datasets recursively under us @@ -671,7 +712,7 @@ class ZfsDataset: # incremental? if prev_snapshot: - cmd.extend(["-i", "@" + prev_snapshot.snapshot_name]) + cmd.extend(["-i", "@" + prev_snapshot.suffix]) cmd.append(self.name) @@ -925,7 +966,7 @@ class ZfsDataset: # target has nothing yet return None else: - for source_snapshot in reversed(self.snapshots): + for source_snapshot in reversed(self.bookmarks): target_snapshot = target_dataset.find_snapshot(source_snapshot) if target_snapshot: if guid_check and source_snapshot.properties['guid'] != target_snapshot.properties['guid']: @@ -1004,7 +1045,7 @@ class ZfsDataset: for target_snapshot in target_dataset.snapshots: if (target_snapshot in target_obsoletes) \ and (not source_common_snapshot or ( - target_snapshot.snapshot_name != source_common_snapshot.snapshot_name)): + target_snapshot.suffix != source_common_snapshot.suffix)): if target_snapshot.exists: target_snapshot.destroy() @@ -1024,7 +1065,7 @@ class ZfsDataset: resume_token = target_dataset.properties['receive_resume_token'] # not valid anymore resume_snapshot = self.get_resume_snapshot(resume_token) - if not resume_snapshot or start_snapshot.snapshot_name != resume_snapshot.snapshot_name: + if not resume_snapshot or start_snapshot.suffix != resume_snapshot.snapshot_name: target_dataset.verbose("Aborting resume, its no longer valid.") target_dataset.abort_resume() else: @@ -1083,7 +1124,7 @@ class ZfsDataset: if (also_other_snapshots or source_snapshot.is_ours()) and not source_snapshot.is_excluded: # create virtual target snapshot target_snapshot = target_dataset.zfs_node.get_dataset( - target_dataset.filesystem_name + "@" + source_snapshot.snapshot_name, force_exists=False) + target_dataset.filesystem_name + "@" + source_snapshot.suffix, force_exists=False) possible_target_snapshots.append(target_snapshot) source_snapshot = self.find_next_snapshot(source_snapshot) @@ -1227,19 +1268,18 @@ class ZfsDataset: resume_token = None - # hold the new common snapshots and release the previous ones + # hold common snapshot on the target. if holds: target_snapshot.hold() - source_snapshot.hold() - - if prev_source_snapshot: - prev_source_snapshot.release() if prev_target_snapshot: prev_target_snapshot.release() - # we may now destroy the previous source snapshot if its obsolete - if prev_source_snapshot in source_obsoletes: + #bookmark common snapshot on source + source_snapshot.bookmark() + + # we may now destroy the previous source snapshot if its obsolete or an bookmark + if prev_source_snapshot and (prev_source_snapshot in source_obsoletes or prev_source_snapshot.is_bookmark): prev_source_snapshot.destroy() # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index 70a89b7..d119b03 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -224,7 +224,7 @@ class ZfsNode(ExecuteNode): # self.debug(datasets) for dataset in datasets: if not dataset.is_changed_ours(min_changed_bytes): - dataset.verbose("No changes since {}".format(dataset.our_snapshots[-1].snapshot_name)) + dataset.verbose("No changes since {}".format(dataset.our_snapshots[-1].suffix)) continue # force_exist, since we're making it From 35f6d5980a43de370e679961d115d6353c80be0e Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 25 Sep 2024 00:09:44 +0200 Subject: [PATCH 020/135] note that this is alpha software! --- zfs_autobackup/CliBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zfs_autobackup/CliBase.py b/zfs_autobackup/CliBase.py index 5aa027c..0c069d2 100644 --- a/zfs_autobackup/CliBase.py +++ b/zfs_autobackup/CliBase.py @@ -10,7 +10,7 @@ class CliBase(object): Overridden in subclasses that add stuff for the specific programs.""" # also used by setup.py - VERSION = "3.4-beta.1" + VERSION = "3.4-alpha" HEADER = "{} v{} - (c)2022 E.H.Eefting (edwin@datux.nl)".format(os.path.basename(sys.argv[0]), VERSION) def __init__(self, argv, print_arguments=True): From 58ad1fa7b130422d1344d49a21c2269fa54ebfa6 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 25 Sep 2024 14:11:40 +0200 Subject: [PATCH 021/135] wip: bookmarks --- tests/test_zfsautobackup34.py | 61 +++++++++++ zfs_autobackup/ZfsDataset.py | 194 ++++++++++++++++++++++------------ zfs_autobackup/ZfsNode.py | 2 +- 3 files changed, 187 insertions(+), 70 deletions(-) create mode 100644 tests/test_zfsautobackup34.py diff --git a/tests/test_zfsautobackup34.py b/tests/test_zfsautobackup34.py new file mode 100644 index 0000000..1ac9f02 --- /dev/null +++ b/tests/test_zfsautobackup34.py @@ -0,0 +1,61 @@ +from basetest import * + +class TestZfsAutobackup32(unittest2.TestCase): + """various new 3.4 features""" + + def setUp(self): + prepare_zpools() + self.longMessage=True + + def test_select_bookmark_or_snapshot(self): + """test if zfs autobackup chooses the most recent common matching dataset when there are both bookmarks and snapshots, some with the wrong GUID""" + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-holds".split(" ")).run()) + + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-holds".split(" ")).run()) + + #OOK VERKEERDE GUID + shelltest("zfs destroy test_source2/fs2/sub@test-20101111000001") + shelltest("zfs destroy test_source1/fs1/sub#test-20101111000001") + + + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-holds --debug".split(" ")).run()) + + + r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) + self.assertMultiLineEqual(r,""" +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000000 +test_source1/fs1@invalid +test_source1/fs1@test-20101111000001 +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000000 +test_source1/fs1/sub@test-20101111000001 +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000000 +test_source2/fs2/sub@test-20101111000001 +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@test-20101111000000 +test_target1/test_source1/fs1@invalid +test_target1/test_source1/fs1@test-20101111000001 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000000 +test_target1/test_source1/fs1/sub@test-20101111000001 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000000 +test_target1/test_source2/fs2/sub@test-20101111000001 +""") + diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index ac74125..f87cf0d 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -29,26 +29,28 @@ class ZfsDataset: self.name = name # full name # caching - self.__snapshots = None # type: None|list[ZfsDataset] + # self.__snapshots = None # type: None|list[ZfsDataset] self.__written_since_ours = None # type: None|int self.__exists_check = None # type: None|bool self.__properties = None # type: None|dict[str,str] self.__recursive_datasets = None # type: None|list[ZfsDataset] self.__datasets = None # type: None|list[ZfsDataset] - self.__bookmarks = None # type: None|list[ZfsDataset] + # self.__bookmarks = None # type: None|list[ZfsDataset] + self.__snapshots_bookmarks = None #type: None|list[ZfsDataset] self.force_exists = force_exists def invalidate_cache(self): """clear caches""" self.force_exists = None - self.__snapshots = None + # self.__snapshots = None self.__written_since_ours = None self.__exists_check = None self.__properties = None self.__recursive_datasets = None self.__datasets = None - self.__bookmarks = None + # self.__bookmarks = None + self.__snapshots_bookmarks = None def __repr__(self): return "{}: {}".format(self.zfs_node, self.name) @@ -121,6 +123,9 @@ class ZfsDataset: if self.is_snapshot: (filesystem, snapshot) = self.name.split("@") return filesystem + elif self.is_bookmark: + (filesystem, snapshot) = self.name.split("#") + return filesystem else: return self.name @@ -136,6 +141,20 @@ class ZfsDataset: raise (Exception("This is not a snapshot or bookmark")) + @property + def typed_suffix(self): + """suffix with @ or # in front of it""" + + if self.is_snapshot: + (filesystem, snapshot_name) = self.name.split("@") + return "@"+snapshot_name + elif self.is_bookmark: + (filesystem, bookmark_name) = self.name.split("#") + return "#"+bookmark_name + + raise (Exception("This is not a snapshot or bookmark")) + + @property def is_snapshot(self): """true if this dataset is a snapshot""" @@ -146,6 +165,11 @@ class ZfsDataset: """true if this dataset is a bookmark""" return self.name.find("#") != -1 + @property + def is_dataset(self): + + return not (self.is_snapshot or self.is_bookmark) + @property def is_excluded(self): @@ -249,44 +273,28 @@ class ZfsDataset: else: return None - # NOTE: unused for now - # def find_prev_snapshot(self, snapshot, also_other_snapshots=False): - # """find previous snapshot in this dataset. None if it doesn't exist. - # - # also_other_snapshots: set to true to also return snapshots that where - # not created by us. (is_ours) - # - # Args: - # :type snapshot: str or ZfsDataset.ZfsDataset - # :type also_other_snapshots: bool - # """ - # - # if self.is_snapshot: - # raise (Exception("Please call this on a dataset.")) - # - # index = self.find_snapshot_index(snapshot) - # while index: - # index = index - 1 - # if also_other_snapshots or self.snapshots[index].is_ours(): - # return self.snapshots[index] - # return None - def find_next_snapshot(self, snapshot): - """find next snapshot in this dataset. None if it doesn't exist + def find_next_snapshot(self, snapshot_bookmark): + """find next snapshot in this dataset, according to snapshot or bookmark. None if it doesn't exist Args: - :type snapshot: ZfsDataset + :type snapshot_bookmark: ZfsDataset """ - if self.is_snapshot: + if not self.is_dataset: raise (Exception("Please call this on a dataset.")) - index = self.find_snapshot_index(snapshot) - while index is not None and index < len(self.snapshots) - 1: - index = index + 1 - return self.snapshots[index] + found=False + for snapshot in self.snapshots_bookmarks: + if snapshot == snapshot_bookmark: + found=True + else: + if found==True and snapshot.is_snapshot: + return snapshot + return None + @property def exists_check(self): """check on disk if it exists""" @@ -458,34 +466,48 @@ class ZfsDataset: return seconds @property - def snapshots(self): - """get all snapshots of this dataset + def snapshots_bookmarks(self): + """get all snapshots and bookmarks of this dataset (ordered by createtxg, so its suitable to determine incremental zfs send order) :rtype: list[ZfsDataset] """ # cached? - if self.__snapshots is None: - self.debug("Getting snapshots") + if self.__snapshots_bookmarks is None: + self.debug("Getting snapshots and bookmarks") cmd = [ - "zfs", "list", "-d", "1", "-r", "-t", "snapshot", "-H", "-o", "name", self.name + "zfs", "list", "-d", "1", "-r", "-t", "snapshot,bookmark", "-H", "-o", "name", "-s", "createtxg", self.name ] - self.__snapshots = self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) + self.__snapshots_bookmarks = self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) - return self.__snapshots + return self.__snapshots_bookmarks - def cache_snapshot(self, snapshot, force=False): - """Update our snapshot cache (if we have any). Use force if you want to force the caching, potentially triggering a zfs list + + @property + def snapshots(self): + """get all snapshots of this dataset + :rtype: list[ZfsDataset] + """ + ret = [] + + for snapshot in self.snapshots_bookmarks: + if snapshot.is_snapshot: + ret.append(snapshot) + + return ret + + def cache_snapshot_bookmark(self, snapshot, force=False): + """Update our snapshot and bookmark cache (if we have any). Use force if you want to force the caching, potentially triggering a zfs list Args: :type snapshot: ZfsDataset """ if force: - self.snapshots.append(snapshot) + self.snapshots_bookmarks.append(snapshot) - elif self.__snapshots is not None: - self.__snapshots.append(snapshot) + elif self.__snapshots_bookmarks is not None: + self.__snapshots_bookmarks.append(snapshot) @property def our_snapshots(self): @@ -512,7 +534,7 @@ class ZfsDataset: return None def find_snapshot(self, snapshot): - """find snapshot by snapshot name (can be a snapshot_name or a different + """find snapshot by snapshot name (can be a suffix or a different ZfsDataset) Returns None if it cant find it. Args: @@ -524,16 +546,40 @@ class ZfsDataset: return None if not isinstance(snapshot, ZfsDataset): - snapshot_name = snapshot + suffix = snapshot else: - snapshot_name = snapshot.suffix + suffix = snapshot.suffix for snapshot in self.snapshots: - if snapshot.suffix == snapshot_name: + if snapshot.suffix == suffix: return snapshot return None + def find_bookmark(self, bookmark): + """find bookmark by bookmark name (can be a suffix or a different + ZfsDataset) Returns None if it cant find it. + + Args: + :rtype: ZfsDataset|None + :type bookmark: str|ZfsDataset|None + """ + + if bookmark is None: + return None + + if not isinstance(bookmark, ZfsDataset): + suffix = bookmark + else: + suffix = bookmark.suffix + + for bookmark in self.bookmarks: + if bookmark.suffix == suffix: + return bookmark + + return None + + def find_snapshot_index(self, snapshot): """find snapshot index by snapshot (can be a snapshot_name or ZfsDataset) @@ -604,21 +650,19 @@ class ZfsDataset: self.zfs_node.run(cmd=cmd) + self.cache_snapshot_bookmark(self.zfs_node.get_dataset( self.name+'#'+self.suffix,force_exists=True)) + @property def bookmarks(self): - if self.__bookmarks is None: - self.debug("Getting bookmarks") + ret = [] - cmd = [ - "zfs", "list", "-d", "1", "-r", "-t", "bookmark", "-H", "-o", "name", self.name - ] + for bookmark in self.snapshots_bookmarks: + if bookmark.is_bookmark: + ret.append(bookmark) - self.__bookmarks = self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) - - - return self.__bookmarks + return ret @property @@ -712,7 +756,8 @@ class ZfsDataset: # incremental? if prev_snapshot: - cmd.extend(["-i", "@" + prev_snapshot.suffix]) + + cmd.extend(["-i", prev_snapshot.typed_suffix]) cmd.append(self.name) @@ -953,8 +998,8 @@ class ZfsDataset: self.snapshots.remove(obsolete) def find_common_snapshot(self, target_dataset, guid_check): - """find latest common snapshot between us and target returns None if its - an initial transfer + """find latest common snapshot/bookmark between us and target returns None if its + an initial transfer. It preffers bookmarks over snapshots on the source side. Target side will always be a snapshots. Args: :rtype: ZfsDataset|None @@ -966,16 +1011,27 @@ class ZfsDataset: # target has nothing yet return None else: - for source_snapshot in reversed(self.bookmarks): - target_snapshot = target_dataset.find_snapshot(source_snapshot) - if target_snapshot: - if guid_check and source_snapshot.properties['guid'] != target_snapshot.properties['guid']: - target_snapshot.warning("Common snapshots have mismatching GUID, ignoring.") + for target_snapshot in reversed(target_dataset.snapshots): + + #Source bookmark? + source_bookmark = self.find_bookmark(target_snapshot) + if source_bookmark: + if guid_check and source_bookmark.properties['guid'] != target_snapshot.properties['guid']: + source_bookmark.warning("Bookmark has mismatching GUID, ignoring.") else: - target_snapshot.debug("common snapshot") + source_bookmark.debug("Common bookmark") + return source_bookmark + + #Source snapshot? + source_snapshot = self.find_snapshot(target_snapshot) + if source_snapshot: + if guid_check and source_snapshot.properties['guid'] != target_snapshot.properties['guid']: + source_snapshot.warning("Snapshot has mismatching GUID, ignoring.") + else: + source_snapshot.debug("Common snapshot") return source_snapshot - # target_dataset.error("Cant find common snapshot with source.") - raise (Exception("Cant find common snapshot with target.")) + + raise (Exception("Cant find common bookmark or snapshot with target.")) def find_incompatible_snapshots(self, common_snapshot, raw): """returns a list[snapshots] that is incompatible for a zfs recv onto diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index d119b03..53f4da5 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -236,7 +236,7 @@ class ZfsNode(ExecuteNode): pools[pool].append(snapshot) - dataset.cache_snapshot(snapshot, force=self.readonly) + dataset.cache_snapshot_bookmark(snapshot, force=self.readonly) if not pools: self.verbose("No changes anywhere: not creating snapshots.") From fbe27f2876df3b67f0f0465cebc1a69bba334c69 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 25 Sep 2024 14:43:55 +0200 Subject: [PATCH 022/135] wip: bookmarks --- tests/test_zfsautobackup34.py | 32 +++++++++++++++++--------------- zfs_autobackup/ZfsDataset.py | 17 +++++++++-------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/tests/test_zfsautobackup34.py b/tests/test_zfsautobackup34.py index 1ac9f02..690f410 100644 --- a/tests/test_zfsautobackup34.py +++ b/tests/test_zfsautobackup34.py @@ -10,52 +10,54 @@ class TestZfsAutobackup32(unittest2.TestCase): def test_select_bookmark_or_snapshot(self): """test if zfs autobackup chooses the most recent common matching dataset when there are both bookmarks and snapshots, some with the wrong GUID""" - with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-holds".split(" ")).run()) - - with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-holds".split(" ")).run()) - #OOK VERKEERDE GUID + shelltest("zfs destroy test_source2/fs2/sub@test-20101111000001") shelltest("zfs destroy test_source1/fs1/sub#test-20101111000001") + #bookmark with incorrect GUID, should fallback to snapshot + shelltest("zfs destroy test_source1/fs1#test-20101111000001") + shelltest("zfs snapshot test_source1/fs1@wrong") + shelltest("zfs bookmark test_source1/fs1@wrong \#test-20101111000001") + shelltest("zfs destroy test_source1/fs1@wrong") with mocktime("20101111000002"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-holds --debug".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-holds".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) self.assertMultiLineEqual(r,""" test_source1 test_source1/fs1 -test_source1/fs1@test-20101111000000 -test_source1/fs1@invalid test_source1/fs1@test-20101111000001 +test_source1/fs1@test-20101111000002 +test_source1/fs1#test-20101111000001 +test_source1/fs1#test-20101111000002 test_source1/fs1/sub -test_source1/fs1/sub@test-20101111000000 test_source1/fs1/sub@test-20101111000001 +test_source1/fs1/sub@test-20101111000002 +test_source1/fs1/sub#test-20101111000002 test_source2 test_source2/fs2 test_source2/fs2/sub -test_source2/fs2/sub@test-20101111000000 -test_source2/fs2/sub@test-20101111000001 +test_source2/fs2/sub@test-20101111000002 +test_source2/fs2/sub#test-20101111000002 test_source2/fs3 test_source2/fs3/sub test_target1 test_target1/test_source1 test_target1/test_source1/fs1 -test_target1/test_source1/fs1@test-20101111000000 -test_target1/test_source1/fs1@invalid test_target1/test_source1/fs1@test-20101111000001 +test_target1/test_source1/fs1@test-20101111000002 test_target1/test_source1/fs1/sub -test_target1/test_source1/fs1/sub@test-20101111000000 test_target1/test_source1/fs1/sub@test-20101111000001 +test_target1/test_source1/fs1/sub@test-20101111000002 test_target1/test_source2 test_target1/test_source2/fs2 test_target1/test_source2/fs2/sub -test_target1/test_source2/fs2/sub@test-20101111000000 test_target1/test_source2/fs2/sub@test-20101111000001 +test_target1/test_source2/fs2/sub@test-20101111000002 """) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index f87cf0d..c55397b 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -59,11 +59,11 @@ class ZfsDataset: return self.name - def __eq__(self, obj): - if not isinstance(obj, ZfsDataset): + def __eq__(self, dataset): + if not isinstance(dataset, ZfsDataset): return False - return self.name == obj.name + return self.name == dataset.name def verbose(self, txt): """ @@ -1089,7 +1089,7 @@ class ZfsDataset: :type target_transfers: list[ZfsDataset] """ - # on source: delete all obsoletes that are not in target_transfers (except common snapshot) + # on source: delete all obsoletes that are not in target_transfers (except common snapshot, if its not a bookmark) for source_snapshot in self.snapshots: if (source_snapshot in source_obsoletes and source_common_snapshot != source_snapshot @@ -1152,10 +1152,10 @@ class ZfsDataset: source_common_snapshot = self.find_common_snapshot(target_dataset, guid_check=guid_check) incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(source_common_snapshot, raw) - # let thinner decide whats obsolete on source after the transfer is done, keeping the last snapshot as common. + # let thinner decide whats obsolete on source after the transfer is done source_obsoletes = [] if self.our_snapshots: - source_obsoletes = self.thin_list(keeps=[self.our_snapshots[-1]])[1] + source_obsoletes = self.thin_list()[1] ### 2: Determine possible target snapshots @@ -1180,11 +1180,12 @@ class ZfsDataset: if (also_other_snapshots or source_snapshot.is_ours()) and not source_snapshot.is_excluded: # create virtual target snapshot target_snapshot = target_dataset.zfs_node.get_dataset( - target_dataset.filesystem_name + "@" + source_snapshot.suffix, force_exists=False) + target_dataset.filesystem_name + source_snapshot.typed_suffix, force_exists=False) possible_target_snapshots.append(target_snapshot) source_snapshot = self.find_next_snapshot(source_snapshot) - ### 3: Let the thinner decide what it wants by looking at all the possible target_snaphots at once + ### 3: Let the thinner decide what it wants by looking at all the possible target_snaphots at once. + # always keep the last target snapshot as common snapshot. if possible_target_snapshots: (target_keeps, target_obsoletes) = target_dataset.zfs_node.thin_list(possible_target_snapshots, keep_snapshots=[ From b839c9a978e833e01b94fd784fe1626464cd1747 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 25 Sep 2024 15:23:37 +0200 Subject: [PATCH 023/135] wip: bookmarks. disable bookmarking if its not supported in on the pools --- tests/test_zfsautobackup34.py | 41 +++++++++++++++++++++++++++++++---- zfs_autobackup/ZfsDataset.py | 11 ++++++++-- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/tests/test_zfsautobackup34.py b/tests/test_zfsautobackup34.py index 690f410..2c5efeb 100644 --- a/tests/test_zfsautobackup34.py +++ b/tests/test_zfsautobackup34.py @@ -1,19 +1,49 @@ from basetest import * -class TestZfsAutobackup32(unittest2.TestCase): +class TestZfsAutobackup34(unittest2.TestCase): """various new 3.4 features""" def setUp(self): prepare_zpools() self.longMessage=True + def test_no_bookmark_source_support(self): + """test if everything is fine when source has no bookmark support (has no features at all even)""" + + subprocess.check_call("zpool destroy test_source1", shell=True) + subprocess.check_call("zpool create -d test_source1 /dev/ram0", shell=True) + subprocess.check_call("zfs create -p test_source1/fs1/sub", shell=True) # recreate with no features at all + subprocess.check_call("zfs set autobackup:test=true test_source1/fs1", shell=True) + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + #should fallback on holds on the source snapshot + r=shelltest("zfs holds test_source1/fs1@test-20101111000001") + self.assertIn("zfs_autobackup:test", r) + + def test_no_bookmark_target_support(self): + """test if everything is fine when target has no bookmark support (has no features at all even)""" + #NOTE: not sure if its ok if only the source supports bookmarks, so currently zfs-autobackup requires both sides to support bookmarks to enable it. + + subprocess.check_call("zpool destroy test_target1", shell=True) + subprocess.check_call("zpool create -d test_target1 /dev/ram2", shell=True) + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + #should fallback on holds on the source snapshot + r=shelltest("zfs holds test_source1/fs1@test-20101111000001") + self.assertIn("zfs_autobackup:test", r) + + def test_select_bookmark_or_snapshot(self): """test if zfs autobackup chooses the most recent common matching dataset when there are both bookmarks and snapshots, some with the wrong GUID""" with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-holds".split(" ")).run()) - + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + #destroy stuff and see if it still selects the correct ones shelltest("zfs destroy test_source2/fs2/sub@test-20101111000001") shelltest("zfs destroy test_source1/fs1/sub#test-20101111000001") @@ -24,7 +54,7 @@ class TestZfsAutobackup32(unittest2.TestCase): shelltest("zfs destroy test_source1/fs1@wrong") with mocktime("20101111000002"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-holds".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -61,3 +91,6 @@ test_target1/test_source2/fs2/sub@test-20101111000001 test_target1/test_source2/fs2/sub@test-20101111000002 """) + #while we're here, check that there are no holds on source common snapshot (since bookmarks replace holds on source side) + r=shelltest("zfs holds test_source2/fs2/sub@test-20101111000002") + self.assertNotIn("zfs_autobackup:test", r) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index c55397b..2d603a6 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1332,8 +1332,15 @@ class ZfsDataset: if prev_target_snapshot: prev_target_snapshot.release() - #bookmark common snapshot on source - source_snapshot.bookmark() + #bookmark common snapshot on source, or use holds if bookmarks are not enabled. + if 'bookmarks' in features: + source_snapshot.bookmark() + else: + if holds: + source_snapshot.hold() + + if prev_source_snapshot: + prev_source_snapshot.release() # we may now destroy the previous source snapshot if its obsolete or an bookmark if prev_source_snapshot and (prev_source_snapshot in source_obsoletes or prev_source_snapshot.is_bookmark): From 7c06e24bd318448b963ee37a292548ec57ecd4c7 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 25 Sep 2024 15:38:58 +0200 Subject: [PATCH 024/135] wip: test --- tests/test_zfsautobackup34.py | 1 + zfs_autobackup/ZfsDataset.py | 2 +- zfs_autobackup/ZfsPool.py | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_zfsautobackup34.py b/tests/test_zfsautobackup34.py index 2c5efeb..80d12ed 100644 --- a/tests/test_zfsautobackup34.py +++ b/tests/test_zfsautobackup34.py @@ -12,6 +12,7 @@ class TestZfsAutobackup34(unittest2.TestCase): subprocess.check_call("zpool destroy test_source1", shell=True) subprocess.check_call("zpool create -d test_source1 /dev/ram0", shell=True) + shelltest("zpool get all test_source1") subprocess.check_call("zfs create -p test_source1/fs1/sub", shell=True) # recreate with no features at all subprocess.check_call("zfs set autobackup:test=true test_source1/fs1", shell=True) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 2d603a6..d80679c 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1333,7 +1333,7 @@ class ZfsDataset: prev_target_snapshot.release() #bookmark common snapshot on source, or use holds if bookmarks are not enabled. - if 'bookmarks' in features: + if 'bookmark_v2' in features: source_snapshot.bookmark() else: if holds: diff --git a/zfs_autobackup/ZfsPool.py b/zfs_autobackup/ZfsPool.py index f0cdf2e..709fb7e 100644 --- a/zfs_autobackup/ZfsPool.py +++ b/zfs_autobackup/ZfsPool.py @@ -1,3 +1,6 @@ +from logging import warning + + class ZfsPool(): """a zfs pool""" From bcd2b69c04d425f054b3dadcb37c7ecdb5213872 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 25 Sep 2024 15:45:36 +0200 Subject: [PATCH 025/135] wip: test --- zfs_autobackup/ZfsDataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index d80679c..cf2f987 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1333,7 +1333,7 @@ class ZfsDataset: prev_target_snapshot.release() #bookmark common snapshot on source, or use holds if bookmarks are not enabled. - if 'bookmark_v2' in features: + if 'bookmark_written' in features: source_snapshot.bookmark() else: if holds: From 865d9602d46b68f5e0f4aaea0117adad47ffe15a Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 25 Sep 2024 15:49:16 +0200 Subject: [PATCH 026/135] wip: bookmarks. disable bookmarking if its not supported in on the pools --- zfs_autobackup/ZfsDataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index cf2f987..abc2626 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1333,6 +1333,7 @@ class ZfsDataset: prev_target_snapshot.release() #bookmark common snapshot on source, or use holds if bookmarks are not enabled. + #NOTE: bookmark_written seems to be needed. (only 'bookmarks' was not enough on ubuntu 20) if 'bookmark_written' in features: source_snapshot.bookmark() else: From 5e74c16c317875e85128326ed32f51df6f5fb5fd Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 1 Oct 2024 22:03:18 +0200 Subject: [PATCH 027/135] --- zfs_autobackup/ZfsDataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index abc2626..c5771c3 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1336,6 +1336,7 @@ class ZfsDataset: #NOTE: bookmark_written seems to be needed. (only 'bookmarks' was not enough on ubuntu 20) if 'bookmark_written' in features: source_snapshot.bookmark() + #note: destroy source_snapshot when obsolete at this point? else: if holds: source_snapshot.hold() From 9faa9efc0c850abca85c36dbfe0cbe619ae326cc Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 1 Oct 2024 22:53:19 +0200 Subject: [PATCH 028/135] wip: bookmarks. allow disabling/reenabling bookmarks --- tests/test_zfsautobackup34.py | 53 +++++++++++++++++++++++++++++++++ zfs_autobackup/ZfsAutobackup.py | 14 ++++++++- zfs_autobackup/ZfsDataset.py | 8 ++--- 3 files changed, 70 insertions(+), 5 deletions(-) diff --git a/tests/test_zfsautobackup34.py b/tests/test_zfsautobackup34.py index 80d12ed..ef17bcf 100644 --- a/tests/test_zfsautobackup34.py +++ b/tests/test_zfsautobackup34.py @@ -95,3 +95,56 @@ test_target1/test_source2/fs2/sub@test-20101111000002 #while we're here, check that there are no holds on source common snapshot (since bookmarks replace holds on source side) r=shelltest("zfs holds test_source2/fs2/sub@test-20101111000002") self.assertNotIn("zfs_autobackup:test", r) + + + def test_disable_bookmarks(self): + """test if we can disable it on an existing backup with bookmarks, with --no-bookmarks and get the old behaviour (holds on source)""" + + #first with bookmarks enabled + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + r=shelltest("zfs list -H -o name -r -t all test_source1") + self.assertMultiLineEqual(r,""" +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000001 +test_source1/fs1#test-20101111000001 +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000001 +test_source1/fs1/sub#test-20101111000001 +""") + + #disable it + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-bookmarks".split(" ")).run()) + + r=shelltest("zfs list -H -o name -r -t all test_source1") + self.assertMultiLineEqual(r,""" +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000001 +test_source1/fs1@test-20101111000002 +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000001 +test_source1/fs1/sub@test-20101111000002 +""") + + #re-enable + with mocktime("20101111000003"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + r=shelltest("zfs list -H -o name -r -t all test_source1") + self.assertMultiLineEqual(r,""" +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000001 +test_source1/fs1@test-20101111000002 +test_source1/fs1@test-20101111000003 +test_source1/fs1#test-20101111000003 +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000001 +test_source1/fs1/sub@test-20101111000002 +test_source1/fs1/sub@test-20101111000003 +test_source1/fs1/sub#test-20101111000003 +""") diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 51e04c2..0534193 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -83,6 +83,9 @@ class ZfsAutobackup(ZfsAuto): help='Don\'t transfer snapshots (useful for cleanups, or if you want a separate send-cronjob)') group.add_argument('--no-holds', action='store_true', help='Don\'t hold snapshots. (Faster. Allows you to destroy common snapshot.)') + group.add_argument('--no-bookmarks', action='store_true', + help='Don\'t use bookmarks.') + group.add_argument('--clear-refreservation', action='store_true', help='Filter "refreservation" property. (recommended, saves space. same as ' '--filter-properties refreservation)') @@ -395,6 +398,15 @@ class ZfsAutobackup(ZfsAuto): target_features = target_node.get_pool(target_dataset).features common_features = source_features and target_features + if self.args.no_bookmarks: + use_bookmarks=False + else: + if not 'bookmark_written' in common_features: + source_dataset.warning("Disabling bookmarks, not supported on both pools.") + use_bookmarks=False + else: + use_bookmarks=True + # sync the snapshots of this dataset source_dataset.sync_snapshots(target_dataset, show_progress=self.args.progress, features=common_features, filter_properties=self.filter_properties_list(), @@ -407,7 +419,7 @@ class ZfsAutobackup(ZfsAuto): send_pipes=send_pipes, recv_pipes=recv_pipes, decrypt=self.args.decrypt, encrypt=self.args.encrypt, zfs_compressed=self.args.zfs_compressed, force=self.args.force, - guid_check=not self.args.no_guid_check) + guid_check=not self.args.no_guid_check, use_bookmarks=use_bookmarks) except Exception as e: fail_count = fail_count + 1 diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index c5771c3..c0b0281 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -131,7 +131,7 @@ class ZfsDataset: @property def suffix(self): - """snapshot or bookmark part of the name""" + """snapshot or bookmark part of the name (the stuff after the # or @) """ if self.is_snapshot: (filesystem, snapshot_name) = self.name.split("@") return snapshot_name @@ -1227,7 +1227,7 @@ class ZfsDataset: def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, - no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check): + no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, use_bookmarks): """sync this dataset's snapshots to target_dataset, while also thinning out old snapshots along the way. @@ -1334,14 +1334,14 @@ class ZfsDataset: #bookmark common snapshot on source, or use holds if bookmarks are not enabled. #NOTE: bookmark_written seems to be needed. (only 'bookmarks' was not enough on ubuntu 20) - if 'bookmark_written' in features: + if use_bookmarks: source_snapshot.bookmark() #note: destroy source_snapshot when obsolete at this point? else: if holds: source_snapshot.hold() - if prev_source_snapshot: + if prev_source_snapshot and prev_source_snapshot.is_snapshot: prev_source_snapshot.release() # we may now destroy the previous source snapshot if its obsolete or an bookmark From 88a4e5276380f7b9286fce339400393b944bd8d7 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 2 Oct 2024 00:42:48 +0200 Subject: [PATCH 029/135] wip: bookmarks. bork --- tests/test_encryption.py | 5 ++++- tests/test_zfsautobackup34.py | 1 + zfs_autobackup/ZfsAutobackup.py | 1 + zfs_autobackup/ZfsDataset.py | 35 ++++++++++++++++++++------------- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/tests/test_encryption.py b/tests/test_encryption.py index 2f08ae3..257ade6 100644 --- a/tests/test_encryption.py +++ b/tests/test_encryption.py @@ -89,12 +89,15 @@ test_target1/test_source2/fs2/sub encryption def test_decrypt(self): """decrypt data and store unencrypted (--decrypt)""" + + self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received".split(" ")).run()) + # NOTE: this also tests if sending a source to a second target works correctly (this failed during bookmark implementation) + self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received --debug".split(" ")).run()) with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(" ")).run()) diff --git a/tests/test_zfsautobackup34.py b/tests/test_zfsautobackup34.py index ef17bcf..5554003 100644 --- a/tests/test_zfsautobackup34.py +++ b/tests/test_zfsautobackup34.py @@ -148,3 +148,4 @@ test_source1/fs1/sub@test-20101111000002 test_source1/fs1/sub@test-20101111000003 test_source1/fs1/sub#test-20101111000003 """) + diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 0534193..cc7e9b7 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -401,6 +401,7 @@ class ZfsAutobackup(ZfsAuto): if self.args.no_bookmarks: use_bookmarks=False else: + # NOTE: bookmark_written seems to be needed. (only 'bookmarks' was not enough on ubuntu 20) if not 'bookmark_written' in common_features: source_dataset.warning("Disabling bookmarks, not supported on both pools.") use_bookmarks=False diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index c0b0281..1079242 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -637,7 +637,7 @@ class ZfsDataset: return True def bookmark(self): - """Bookmark this snapshot""" + """Bookmark this snapshot, and return the bookmark""" if not self.is_snapshot: raise(Exception("Can only bookmark a snapshot!")) @@ -650,8 +650,9 @@ class ZfsDataset: self.zfs_node.run(cmd=cmd) - self.cache_snapshot_bookmark(self.zfs_node.get_dataset( self.name+'#'+self.suffix,force_exists=True)) - + bookmark=self.zfs_node.get_dataset( self.name+'#'+self.suffix,force_exists=True) + self.cache_snapshot_bookmark(bookmark) + return bookmark @property def bookmarks(self): @@ -1303,7 +1304,7 @@ class ZfsDataset: # now actually transfer the snapshots do_rollback = rollback - prev_source_snapshot = source_common_snapshot + prev_source_snapshot_bookmark = source_common_snapshot prev_target_snapshot = target_dataset.find_snapshot(source_common_snapshot) for target_snapshot in target_transfers: @@ -1315,7 +1316,7 @@ class ZfsDataset: do_rollback = False source_snapshot.transfer_snapshot(target_snapshot, features=features, - prev_snapshot=prev_source_snapshot, show_progress=show_progress, + prev_snapshot=prev_source_snapshot_bookmark, show_progress=show_progress, filter_properties=active_filter_properties, set_properties=active_set_properties, ignore_recv_exit_code=ignore_recv_exit_code, @@ -1325,7 +1326,7 @@ class ZfsDataset: resume_token = None - # hold common snapshot on the target. + # hold/release common snapshot on the target. if holds: target_snapshot.hold() @@ -1333,30 +1334,36 @@ class ZfsDataset: prev_target_snapshot.release() #bookmark common snapshot on source, or use holds if bookmarks are not enabled. - #NOTE: bookmark_written seems to be needed. (only 'bookmarks' was not enough on ubuntu 20) if use_bookmarks: - source_snapshot.bookmark() + source_bookmark=source_snapshot.bookmark() #note: destroy source_snapshot when obsolete at this point? else: + source_bookmark=None if holds: source_snapshot.hold() - if prev_source_snapshot and prev_source_snapshot.is_snapshot: - prev_source_snapshot.release() + if prev_source_snapshot_bookmark and prev_source_snapshot_bookmark.is_snapshot: + prev_source_snapshot_bookmark.release() # we may now destroy the previous source snapshot if its obsolete or an bookmark - if prev_source_snapshot and (prev_source_snapshot in source_obsoletes or prev_source_snapshot.is_bookmark): - prev_source_snapshot.destroy() + #FIXME: met bookmarks kan de huidige snapshot na send ook meteen weg + #FIXME: klopt niet, nu haalt ie altijd bookmark weg? wat als we naar andere target willen senden (zoals in test_encryption.py) + if prev_source_snapshot_bookmark and (prev_source_snapshot_bookmark in source_obsoletes or prev_source_snapshot_bookmark.is_bookmark): + prev_source_snapshot_bookmark.destroy() # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, # the rest was already destroyed or will not be send) if prev_target_snapshot in target_obsoletes: prev_target_snapshot.destroy() - prev_source_snapshot = source_snapshot + #we always try to use the bookmark during incremental send + if source_bookmark: + prev_source_snapshot_bookmark = source_bookmark + else: + prev_source_snapshot_bookmark = source_snapshot + prev_target_snapshot = target_snapshot - # source_snapshot = self.find_next_snapshot(source_snapshot, also_other_snapshots) def mount(self, mount_point): From ec9ca29620de7b59202c1ea34a1cc9b206ad3ce8 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 2 Oct 2024 16:17:56 +0200 Subject: [PATCH 030/135] added tag support (needed for bookmarks). implements #151 --- zfs_autobackup/ZfsAuto.py | 29 ++++++++++- zfs_autobackup/ZfsAutobackup.py | 25 +++++----- zfs_autobackup/ZfsDataset.py | 85 ++++++++++++++++++--------------- zfs_autobackup/ZfsNode.py | 13 +++-- 4 files changed, 98 insertions(+), 54 deletions(-) diff --git a/zfs_autobackup/ZfsAuto.py b/zfs_autobackup/ZfsAuto.py index 58ecc66..d03b124 100644 --- a/zfs_autobackup/ZfsAuto.py +++ b/zfs_autobackup/ZfsAuto.py @@ -57,6 +57,7 @@ class ZfsAuto(CliBase): self.property_name = args.property_format.format(args.backup_name) self.snapshot_time_format = args.snapshot_format.format(args.backup_name) self.hold_name = args.hold_format.format(args.backup_name) + self.tag_seperator = args.tag_seperator dt = datetime_now(args.utc) @@ -68,6 +69,27 @@ class ZfsAuto(CliBase): self.verbose("Snapshot format : {}".format(self.snapshot_time_format)) self.verbose("Timezone : {}".format("UTC" if args.utc else "Local")) + seperator_test = datetime_now(False).strftime(self.snapshot_time_format) + + # according to man 8 zfs: + valid_tags = "_.: -" + if self.tag_seperator not in valid_tags or self.tag_seperator == '': + self.log.error("Invalid tag seperator. Allowed: '{}'".format(valid_tags)) + sys.exit(255) + + if self.tag_seperator in seperator_test: + self.log.error("Tag seperator '{}' may not be used in snapshot format: {}".format(self.tag_seperator, + self.snapshot_time_format)) + sys.exit(255) + + if args.tag and self.tag_seperator in args.tag: + self.log.error( + "Tag '{}' may not contain tag seperator '{}'".format(args.tag, self.tag_seperator)) + sys.exit(255) + + if args.tag: + self.verbose("Tag : {}".format(self.tag_seperator + args.tag)) + return args def get_parser(self): @@ -98,13 +120,18 @@ class ZfsAuto(CliBase): help='ZFS hold string format. Default: %(default)s') group.add_argument('--strip-path', metavar='N', default=0, type=int, help='Number of directories to strip from target path.') + group.add_argument('--tag-seperator', metavar='CHAR', default="_", + help="Tag seperator for snapshots and bookmarks. Default: %(default)s") + group.add_argument('--tag', metavar='TAG', default=None, + help='Backup tag to add to snapshots names. (For administrative purposes)') group = parser.add_argument_group("Selection options") group.add_argument('--ignore-replicated', action='store_true', help=argparse.SUPPRESS) group.add_argument('--exclude-unchanged', metavar='BYTES', default=0, type=int, help='Exclude datasets that have less than BYTES data changed since any last snapshot. (Use with proxmox HA replication)') group.add_argument('--exclude-received', action='store_true', - help='Exclude datasets that have the origin of their autobackup: property as "received".' , ) + help='Exclude datasets that have the origin of their autobackup: property as "received".', ) + # group.add_argument('--include-received', action='store_true', # help=argparse.SUPPRESS) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index cc7e9b7..70a67a7 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -48,9 +48,8 @@ class ZfsAutobackup(ZfsAuto): self.warning("Using --compress with --zfs-compressed, might be inefficient.") if args.decrypt: - self.warning("Properties will not be sent over for datasets that will be decrypted. (zfs bug https://github.com/openzfs/zfs/issues/16275)") - - + self.warning( + "Properties will not be sent over for datasets that will be decrypted. (zfs bug https://github.com/openzfs/zfs/issues/16275)") return args @@ -127,8 +126,8 @@ class ZfsAutobackup(ZfsAuto): help='Limit data transfer rate in Bytes/sec (e.g. 128K. requires mbuffer.)') group.add_argument('--buffer', metavar='SIZE', default=None, help='Add zfs send and recv buffers to smooth out IO bursts. (e.g. 128M. requires mbuffer)') - parser.add_argument('--buffer-chunk-size', metavar="BUFFERCHUNKSIZE", default=None, - help='Tune chunk size when mbuffer is used. (requires mbuffer.)') + group.add_argument('--buffer-chunk-size', metavar="BUFFERCHUNKSIZE", default=None, + help='Tune chunk size when mbuffer is used. (requires mbuffer.)') group.add_argument('--send-pipe', metavar="COMMAND", default=[], action='append', help='pipe zfs send output through COMMAND (can be used multiple times)') group.add_argument('--recv-pipe', metavar="COMMAND", default=[], action='append', @@ -399,14 +398,14 @@ class ZfsAutobackup(ZfsAuto): common_features = source_features and target_features if self.args.no_bookmarks: - use_bookmarks=False + use_bookmarks = False else: # NOTE: bookmark_written seems to be needed. (only 'bookmarks' was not enough on ubuntu 20) if not 'bookmark_written' in common_features: source_dataset.warning("Disabling bookmarks, not supported on both pools.") - use_bookmarks=False + use_bookmarks = False else: - use_bookmarks=True + use_bookmarks = True # sync the snapshots of this dataset source_dataset.sync_snapshots(target_dataset, show_progress=self.args.progress, @@ -455,7 +454,6 @@ class ZfsAutobackup(ZfsAuto): if self.args.clear_refreservation: filter_properties.append("refreservation") - return filter_properties def set_properties_list(self): @@ -496,7 +494,8 @@ class ZfsAutobackup(ZfsAuto): ssh_config=self.args.ssh_config, ssh_to=self.args.ssh_source, readonly=self.args.test, debug_output=self.args.debug_output, description=description, thinner=source_thinner, - exclude_snapshot_patterns=self.args.exclude_snapshot_pattern) + exclude_snapshot_patterns=self.args.exclude_snapshot_pattern, + tag_seperator=self.tag_seperator) ################# select source datasets self.set_title("Selecting") @@ -512,6 +511,9 @@ class ZfsAutobackup(ZfsAuto): if not self.args.no_snapshot: self.set_title("Snapshotting") snapshot_name = datetime_now(self.args.utc).strftime(self.snapshot_time_format) + if self.args.tag: + snapshot_name = snapshot_name + self.tag_seperator + self.args.tag + source_node.consistent_snapshot(source_datasets, snapshot_name, min_changed_bytes=self.args.min_change, pre_snapshot_cmds=self.args.pre_snapshot_cmd, @@ -534,7 +536,8 @@ class ZfsAutobackup(ZfsAuto): ssh_to=self.args.ssh_target, readonly=self.args.test, debug_output=self.args.debug_output, description="[Target]", - thinner=target_thinner) + exclude_snapshot_patterns=self.args.exclude_snapshot_pattern, + thinner=target_thinner, tag_seperator=self.tag_seperator) target_node.verbose("Receive datasets under: {}".format(self.args.target_path)) self.set_title("Synchronising") diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 1079242..28bcd90 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -6,6 +6,8 @@ import time from .ExecuteNode import ExecuteError +# NOTE: get/create instances via zfs_node.get_dataset(). This is to make sure there is only one ZfsDataset object per actual dataset. + class ZfsDataset: """a zfs dataset (filesystem/volume/snapshot/clone) Note that a dataset doesn't have to actually exist (yet/anymore) Also most properties are cached @@ -26,7 +28,9 @@ class ZfsDataset: :type force_exists: bool """ self.zfs_node = zfs_node - self.name = name # full name + self.name = name # full actual name of dataset + + self.force_exists = force_exists # caching # self.__snapshots = None # type: None|list[ZfsDataset] @@ -36,9 +40,7 @@ class ZfsDataset: self.__recursive_datasets = None # type: None|list[ZfsDataset] self.__datasets = None # type: None|list[ZfsDataset] # self.__bookmarks = None # type: None|list[ZfsDataset] - self.__snapshots_bookmarks = None #type: None|list[ZfsDataset] - - self.force_exists = force_exists + self.__snapshots_bookmarks = None # type: None|list[ZfsDataset] def invalidate_cache(self): """clear caches""" @@ -141,20 +143,29 @@ class ZfsDataset: raise (Exception("This is not a snapshot or bookmark")) + @property + def tagless_suffix(self): + """snapshot or bookmark part of the name, but without the tag.""" + + suffix = self.suffix + if self.zfs_node.tag_seperator in suffix: + return suffix.split(self.zfs_node.tag_seperator)[0] + else: + return suffix + @property def typed_suffix(self): """suffix with @ or # in front of it""" if self.is_snapshot: (filesystem, snapshot_name) = self.name.split("@") - return "@"+snapshot_name + return "@" + snapshot_name elif self.is_bookmark: (filesystem, bookmark_name) = self.name.split("#") - return "#"+bookmark_name + return "#" + bookmark_name raise (Exception("This is not a snapshot or bookmark")) - @property def is_snapshot(self): """true if this dataset is a snapshot""" @@ -170,15 +181,14 @@ class ZfsDataset: return not (self.is_snapshot or self.is_bookmark) - @property - def is_excluded(self): + def is_snapshot_excluded(self): """true if this dataset is a snapshot and matches the exclude pattern""" if not self.is_snapshot: return False for pattern in self.zfs_node.exclude_snapshot_patterns: - if pattern.search(self.name) is not None: + if pattern.search(self.suffix) is not None: self.debug("Excluded (path matches snapshot exclude pattern)") return True @@ -273,7 +283,6 @@ class ZfsDataset: else: return None - def find_next_snapshot(self, snapshot_bookmark): """find next snapshot in this dataset, according to snapshot or bookmark. None if it doesn't exist @@ -284,17 +293,16 @@ class ZfsDataset: if not self.is_dataset: raise (Exception("Please call this on a dataset.")) - found=False + found = False for snapshot in self.snapshots_bookmarks: if snapshot == snapshot_bookmark: - found=True + found = True else: - if found==True and snapshot.is_snapshot: + if found == True and snapshot.is_snapshot: return snapshot return None - @property def exists_check(self): """check on disk if it exists""" @@ -441,11 +449,13 @@ class ZfsDataset: """get timestamp from snapshot name. Only works for our own snapshots with the correct format. Snapshots that are not ours always return None + Note that the tag-part in the name is ignored, so snapsnots are ours regardless of their tag. + :rtype: int|None """ try: - dt = datetime.strptime(self.suffix, self.zfs_node.snapshot_time_format) + dt = datetime.strptime(self.tagless_suffix, self.zfs_node.snapshot_time_format) except ValueError: return None @@ -476,14 +486,15 @@ class ZfsDataset: self.debug("Getting snapshots and bookmarks") cmd = [ - "zfs", "list", "-d", "1", "-r", "-t", "snapshot,bookmark", "-H", "-o", "name", "-s", "createtxg", self.name + "zfs", "list", "-d", "1", "-r", "-t", "snapshot,bookmark", "-H", "-o", "name", "-s", "createtxg", + self.name ] - self.__snapshots_bookmarks = self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) + self.__snapshots_bookmarks = self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), + force_exists=True) return self.__snapshots_bookmarks - @property def snapshots(self): """get all snapshots of this dataset @@ -579,7 +590,6 @@ class ZfsDataset: return None - def find_snapshot_index(self, snapshot): """find snapshot index by snapshot (can be a snapshot_name or ZfsDataset) @@ -640,17 +650,17 @@ class ZfsDataset: """Bookmark this snapshot, and return the bookmark""" if not self.is_snapshot: - raise(Exception("Can only bookmark a snapshot!")) + raise (Exception("Can only bookmark a snapshot!")) self.debug("Bookmarking") cmd = [ - "zfs", "bookmark", self.name, "#"+self.suffix + "zfs", "bookmark", self.name, "#" + self.suffix ] self.zfs_node.run(cmd=cmd) - bookmark=self.zfs_node.get_dataset( self.name+'#'+self.suffix,force_exists=True) + bookmark = self.zfs_node.get_dataset(self.name + '#' + self.suffix, force_exists=True) self.cache_snapshot_bookmark(bookmark) return bookmark @@ -665,7 +675,6 @@ class ZfsDataset: return ret - @property def recursive_datasets(self, types="filesystem,volume"): """get all (non-snapshot) datasets recursively under us @@ -757,7 +766,6 @@ class ZfsDataset: # incremental? if prev_snapshot: - cmd.extend(["-i", prev_snapshot.typed_suffix]) cmd.append(self.name) @@ -1014,7 +1022,7 @@ class ZfsDataset: else: for target_snapshot in reversed(target_dataset.snapshots): - #Source bookmark? + # Source bookmark? source_bookmark = self.find_bookmark(target_snapshot) if source_bookmark: if guid_check and source_bookmark.properties['guid'] != target_snapshot.properties['guid']: @@ -1023,7 +1031,7 @@ class ZfsDataset: source_bookmark.debug("Common bookmark") return source_bookmark - #Source snapshot? + # Source snapshot? source_snapshot = self.find_snapshot(target_snapshot) if source_snapshot: if guid_check and source_snapshot.properties['guid'] != target_snapshot.properties['guid']: @@ -1178,7 +1186,7 @@ class ZfsDataset: while source_snapshot: # we want it? - if (also_other_snapshots or source_snapshot.is_ours()) and not source_snapshot.is_excluded: + if (also_other_snapshots or source_snapshot.is_ours()) and not source_snapshot.is_snapshot_excluded: # create virtual target snapshot target_snapshot = target_dataset.zfs_node.get_dataset( target_dataset.filesystem_name + source_snapshot.typed_suffix, force_exists=False) @@ -1228,7 +1236,8 @@ class ZfsDataset: def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, - no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, use_bookmarks): + no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, + use_bookmarks): """sync this dataset's snapshots to target_dataset, while also thinning out old snapshots along the way. @@ -1333,12 +1342,12 @@ class ZfsDataset: if prev_target_snapshot: prev_target_snapshot.release() - #bookmark common snapshot on source, or use holds if bookmarks are not enabled. + # bookmark common snapshot on source, or use holds if bookmarks are not enabled. if use_bookmarks: - source_bookmark=source_snapshot.bookmark() - #note: destroy source_snapshot when obsolete at this point? + source_bookmark = source_snapshot.bookmark() + # note: destroy source_snapshot when obsolete at this point? else: - source_bookmark=None + source_bookmark = None if holds: source_snapshot.hold() @@ -1346,9 +1355,10 @@ class ZfsDataset: prev_source_snapshot_bookmark.release() # we may now destroy the previous source snapshot if its obsolete or an bookmark - #FIXME: met bookmarks kan de huidige snapshot na send ook meteen weg - #FIXME: klopt niet, nu haalt ie altijd bookmark weg? wat als we naar andere target willen senden (zoals in test_encryption.py) - if prev_source_snapshot_bookmark and (prev_source_snapshot_bookmark in source_obsoletes or prev_source_snapshot_bookmark.is_bookmark): + # FIXME: met bookmarks kan de huidige snapshot na send ook meteen weg + # FIXME: klopt niet, nu haalt ie altijd bookmark weg? wat als we naar andere target willen senden (zoals in test_encryption.py) + if prev_source_snapshot_bookmark and ( + prev_source_snapshot_bookmark in source_obsoletes or prev_source_snapshot_bookmark.is_bookmark): prev_source_snapshot_bookmark.destroy() # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, @@ -1356,7 +1366,7 @@ class ZfsDataset: if prev_target_snapshot in target_obsoletes: prev_target_snapshot.destroy() - #we always try to use the bookmark during incremental send + # we always try to use the bookmark during incremental send if source_bookmark: prev_source_snapshot_bookmark = source_bookmark else: @@ -1364,7 +1374,6 @@ class ZfsDataset: prev_target_snapshot = target_snapshot - def mount(self, mount_point): self.debug("Mounting") diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index 53f4da5..be9b8b2 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -17,13 +17,18 @@ from .util import datetime_now class ZfsNode(ExecuteNode): """a node that contains zfs datasets. implements global (systemwide/pool wide) zfs commands""" - def __init__(self, logger, utc=False, snapshot_time_format="", hold_name="", ssh_config=None, ssh_to=None, - readonly=False, - description="", - debug_output=False, thinner=None, exclude_snapshot_patterns=[]): + # def __init__(self, logger, utc=False, snapshot_time_format="", hold_name="", ssh_config=None, ssh_to=None, + # readonly=False, + # description="", + # debug_output=False, thinner=None, exclude_snapshot_patterns=None, tag_seperator='~'): + def __init__(self, logger, utc, snapshot_time_format, hold_name, ssh_config, ssh_to, + readonly, + description, + debug_output, thinner, exclude_snapshot_patterns, tag_seperator): self.utc = utc self.snapshot_time_format = snapshot_time_format + self.tag_seperator = tag_seperator self.hold_name = hold_name self.description = description From 0715a45f11c34ce8d3472ddafd8222aa7a5e2d2e Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Thu, 3 Oct 2024 15:20:00 +0200 Subject: [PATCH 031/135] test tags --- tests/test_zfsautobackup34.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_zfsautobackup34.py b/tests/test_zfsautobackup34.py index 5554003..81ebbf4 100644 --- a/tests/test_zfsautobackup34.py +++ b/tests/test_zfsautobackup34.py @@ -147,5 +147,35 @@ test_source1/fs1/sub@test-20101111000001 test_source1/fs1/sub@test-20101111000002 test_source1/fs1/sub@test-20101111000003 test_source1/fs1/sub#test-20101111000003 +""") + + def test_tags(self): + + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test1".split(" ")).run()) + + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test2".split(" ")).run()) + + with mocktime("20101111000003"): + # make sure the thinner sees and cleans up the old snaphots that have a tag + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --keep-source=2 --keep-target=2".split(" ")).run()) + + r=shelltest("zfs list -H -r -t snapshot -o name "+TEST_POOLS) + + self.assertMultiLineEqual(r,""" +test_source1/fs1@test-20101111000002_test2 +test_source1/fs1@test-20101111000003 +test_source1/fs1/sub@test-20101111000002_test2 +test_source1/fs1/sub@test-20101111000003 +test_source2/fs2/sub@test-20101111000002_test2 +test_source2/fs2/sub@test-20101111000003 +test_target1/test_source1/fs1@test-20101111000002_test2 +test_target1/test_source1/fs1@test-20101111000003 +test_target1/test_source1/fs1/sub@test-20101111000002_test2 +test_target1/test_source1/fs1/sub@test-20101111000003 +test_target1/test_source2/fs2/sub@test-20101111000002_test2 +test_target1/test_source2/fs2/sub@test-20101111000003 """) From 2f4ea79ff3b4543abf4e2ac4108f7f914ccc16a1 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Sat, 5 Oct 2024 17:15:36 +0200 Subject: [PATCH 032/135] wip --- tests/test_zfsautobackup34.py | 99 ++++++++++++++++++++++----------- zfs_autobackup/ZfsAutobackup.py | 8 ++- zfs_autobackup/ZfsDataset.py | 14 +++-- 3 files changed, 80 insertions(+), 41 deletions(-) diff --git a/tests/test_zfsautobackup34.py b/tests/test_zfsautobackup34.py index 81ebbf4..8e64d8b 100644 --- a/tests/test_zfsautobackup34.py +++ b/tests/test_zfsautobackup34.py @@ -1,11 +1,12 @@ from basetest import * + class TestZfsAutobackup34(unittest2.TestCase): """various new 3.4 features""" def setUp(self): prepare_zpools() - self.longMessage=True + self.longMessage = True def test_no_bookmark_source_support(self): """test if everything is fine when source has no bookmark support (has no features at all even)""" @@ -13,19 +14,19 @@ class TestZfsAutobackup34(unittest2.TestCase): subprocess.check_call("zpool destroy test_source1", shell=True) subprocess.check_call("zpool create -d test_source1 /dev/ram0", shell=True) shelltest("zpool get all test_source1") - subprocess.check_call("zfs create -p test_source1/fs1/sub", shell=True) # recreate with no features at all + subprocess.check_call("zfs create -p test_source1/fs1/sub", shell=True) # recreate with no features at all subprocess.check_call("zfs set autobackup:test=true test_source1/fs1", shell=True) with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - #should fallback on holds on the source snapshot - r=shelltest("zfs holds test_source1/fs1@test-20101111000001") + # should fallback on holds on the source snapshot + r = shelltest("zfs holds test_source1/fs1@test-20101111000001") self.assertIn("zfs_autobackup:test", r) def test_no_bookmark_target_support(self): """test if everything is fine when target has no bookmark support (has no features at all even)""" - #NOTE: not sure if its ok if only the source supports bookmarks, so currently zfs-autobackup requires both sides to support bookmarks to enable it. + # NOTE: not sure if its ok if only the source supports bookmarks, so currently zfs-autobackup requires both sides to support bookmarks to enable it. subprocess.check_call("zpool destroy test_target1", shell=True) subprocess.check_call("zpool create -d test_target1 /dev/ram2", shell=True) @@ -33,22 +34,21 @@ class TestZfsAutobackup34(unittest2.TestCase): with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - #should fallback on holds on the source snapshot - r=shelltest("zfs holds test_source1/fs1@test-20101111000001") + # should fallback on holds on the source snapshot + r = shelltest("zfs holds test_source1/fs1@test-20101111000001") self.assertIn("zfs_autobackup:test", r) - def test_select_bookmark_or_snapshot(self): """test if zfs autobackup chooses the most recent common matching dataset when there are both bookmarks and snapshots, some with the wrong GUID""" with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - #destroy stuff and see if it still selects the correct ones + # destroy stuff and see if it still selects the correct ones shelltest("zfs destroy test_source2/fs2/sub@test-20101111000001") shelltest("zfs destroy test_source1/fs1/sub#test-20101111000001") - #bookmark with incorrect GUID, should fallback to snapshot + # bookmark with incorrect GUID, should fallback to snapshot shelltest("zfs destroy test_source1/fs1#test-20101111000001") shelltest("zfs snapshot test_source1/fs1@wrong") shelltest("zfs bookmark test_source1/fs1@wrong \#test-20101111000001") @@ -57,9 +57,8 @@ class TestZfsAutobackup34(unittest2.TestCase): with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000001 @@ -92,20 +91,19 @@ test_target1/test_source2/fs2/sub@test-20101111000001 test_target1/test_source2/fs2/sub@test-20101111000002 """) - #while we're here, check that there are no holds on source common snapshot (since bookmarks replace holds on source side) - r=shelltest("zfs holds test_source2/fs2/sub@test-20101111000002") + # while we're here, check that there are no holds on source common snapshot (since bookmarks replace holds on source side) + r = shelltest("zfs holds test_source2/fs2/sub@test-20101111000002") self.assertNotIn("zfs_autobackup:test", r) - def test_disable_bookmarks(self): """test if we can disable it on an existing backup with bookmarks, with --no-bookmarks and get the old behaviour (holds on source)""" - #first with bookmarks enabled + # first with bookmarks enabled with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all test_source1") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t all test_source1") + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000001 @@ -115,12 +113,13 @@ test_source1/fs1/sub@test-20101111000001 test_source1/fs1/sub#test-20101111000001 """) - #disable it + # disable it with mocktime("20101111000002"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-bookmarks".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --no-bookmarks".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all test_source1") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t all test_source1") + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000001 @@ -130,12 +129,12 @@ test_source1/fs1/sub@test-20101111000001 test_source1/fs1/sub@test-20101111000002 """) - #re-enable + # re-enable with mocktime("20101111000003"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all test_source1") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t all test_source1") + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000001 @@ -150,21 +149,23 @@ test_source1/fs1/sub#test-20101111000003 """) def test_tags(self): - - with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test1".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test1".split(" ")).run()) with mocktime("20101111000002"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test2".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test2".split(" ")).run()) with mocktime("20101111000003"): # make sure the thinner sees and cleans up the old snaphots that have a tag - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --keep-source=2 --keep-target=2".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --keep-source=2 --keep-target=2".split( + " ")).run()) - r=shelltest("zfs list -H -r -t snapshot -o name "+TEST_POOLS) + r = shelltest("zfs list -H -r -t snapshot -o name " + TEST_POOLS) - self.assertMultiLineEqual(r,""" + self.assertMultiLineEqual(r, """ test_source1/fs1@test-20101111000002_test2 test_source1/fs1@test-20101111000003 test_source1/fs1/sub@test-20101111000002_test2 @@ -179,3 +180,35 @@ test_target1/test_source2/fs2/sub@test-20101111000002_test2 test_target1/test_source2/fs2/sub@test-20101111000003 """) + def test_double_send_bookmark(self): + """test sending the same snaphots to 2 targets, and check if they each use their own bookmark and delete them correctly.""" + + shelltest("zfs create test_target1/a") + shelltest("zfs create test_target1/b") + + # full + with mocktime("20101111000001"): + self.assertFalse( + ZfsAutobackup( + "test test_target1/a --no-progress --verbose --allow-empty --tag tag1".split(" ")).run()) + + # increment, should be from bookmark + with mocktime("20101111000002"): + self.assertFalse( + ZfsAutobackup( + "test test_target1/a --no-progress --verbose --allow-empty --tag tag1".split(" ")).run()) + + # to target b, now each has one full + two incrementals, which should be from their own bookmarks. + with mocktime("20101111000003"): + self.assertFalse( + ZfsAutobackup( + "test test_target1/b --no-progress --verbose --no-snapshot --allow-empty".split( + " ")).run()) + + # result: + # for target a the bookmarks should be at 20101111000002, for target b the bookmarks should be at 20101111000003 + r = shelltest("zfs list -H -r -t all -o name " + TEST_POOLS) + + self.assertMultiLineEqual(r, """ +... +""") diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 70a67a7..234dc37 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -357,8 +357,9 @@ class ZfsAutobackup(ZfsAuto): target_datasets[target_name] = source_dataset # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: - def sync_datasets(self, source_node, source_datasets, target_node): + def sync_datasets(self, source_node, source_datasets, target_node, bookmark_tag): """Sync datasets, or thin-only on both sides + :type bookmark_tag: str :type target_node: ZfsNode :type source_datasets: list of ZfsDataset :type source_node: ZfsNode @@ -419,7 +420,8 @@ class ZfsAutobackup(ZfsAuto): send_pipes=send_pipes, recv_pipes=recv_pipes, decrypt=self.args.decrypt, encrypt=self.args.encrypt, zfs_compressed=self.args.zfs_compressed, force=self.args.force, - guid_check=not self.args.no_guid_check, use_bookmarks=use_bookmarks) + guid_check=not self.args.no_guid_check, use_bookmarks=use_bookmarks, + bookmark_tag=bookmark_tag) except Exception as e: fail_count = fail_count + 1 @@ -556,7 +558,7 @@ class ZfsAutobackup(ZfsAuto): fail_count = self.sync_datasets( source_node=source_node, source_datasets=source_datasets, - target_node=target_node) + target_node=target_node, bookmark_tag=target_dataset.properties['guid']) # no target specified, run in snapshot-only mode else: diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 28bcd90..3c307ff 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -646,8 +646,9 @@ class ZfsDataset: return True - def bookmark(self): - """Bookmark this snapshot, and return the bookmark""" + def bookmark(self, tag): + """Bookmark this snapshot, and return the bookmark.""" + # NOTE: we use the tag to add the target_path GUID, so that we can have multiple bookmarks for the same snapshot, but for different target. This is to make sure you can send a backup to two locations, without them interfering with eachothers bookmarks. if not self.is_snapshot: raise (Exception("Can only bookmark a snapshot!")) @@ -655,7 +656,7 @@ class ZfsDataset: self.debug("Bookmarking") cmd = [ - "zfs", "bookmark", self.name, "#" + self.suffix + "zfs", "bookmark", self.name, "#" + self.tagless_suffix + self.zfs_node.tag_seperator + tag ] self.zfs_node.run(cmd=cmd) @@ -772,6 +773,7 @@ class ZfsDataset: cmd.extend(send_pipes) + self.error(cmd) output_pipe = self.zfs_node.run(cmd, pipe=True, readonly=True) return output_pipe @@ -1237,7 +1239,7 @@ class ZfsDataset: def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, - use_bookmarks): + use_bookmarks, bookmark_tag): """sync this dataset's snapshots to target_dataset, while also thinning out old snapshots along the way. @@ -1256,6 +1258,8 @@ class ZfsDataset: :type also_other_snapshots: bool :type no_send: bool :type guid_check: bool + :type use_bookmarks: bool + :type bookmark_tag: str """ # self.verbose("-> {}".format(target_dataset)) @@ -1344,7 +1348,7 @@ class ZfsDataset: # bookmark common snapshot on source, or use holds if bookmarks are not enabled. if use_bookmarks: - source_bookmark = source_snapshot.bookmark() + source_bookmark = source_snapshot.bookmark(bookmark_tag) # note: destroy source_snapshot when obsolete at this point? else: source_bookmark = None From d283da88751ae58918f0ed5b374bca388ecc7d26 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Sun, 6 Oct 2024 11:19:03 +0200 Subject: [PATCH 033/135] wip --- zfs_autobackup/ZfsAutobackup.py | 9 ++++--- zfs_autobackup/ZfsDataset.py | 46 +++++++++++++++++++++------------ zfs_autobackup/ZfsNode.py | 1 + 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 234dc37..dbafadc 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -357,9 +357,9 @@ class ZfsAutobackup(ZfsAuto): target_datasets[target_name] = source_dataset # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: - def sync_datasets(self, source_node, source_datasets, target_node, bookmark_tag): + def sync_datasets(self, source_node, source_datasets, target_node, bookmark_name): """Sync datasets, or thin-only on both sides - :type bookmark_tag: str + :type bookmark_name: str :type target_node: ZfsNode :type source_datasets: list of ZfsDataset :type source_node: ZfsNode @@ -421,7 +421,7 @@ class ZfsAutobackup(ZfsAuto): decrypt=self.args.decrypt, encrypt=self.args.encrypt, zfs_compressed=self.args.zfs_compressed, force=self.args.force, guid_check=not self.args.no_guid_check, use_bookmarks=use_bookmarks, - bookmark_tag=bookmark_tag) + bookmark_name=bookmark_name) except Exception as e: fail_count = fail_count + 1 @@ -558,7 +558,8 @@ class ZfsAutobackup(ZfsAuto): fail_count = self.sync_datasets( source_node=source_node, source_datasets=source_datasets, - target_node=target_node, bookmark_tag=target_dataset.properties['guid']) + target_node=target_node, + bookmark_name=self.args.backup_name + self.tag_seperator + target_dataset.properties['guid']) # no target specified, run in snapshot-only mode else: diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 3c307ff..51f6a37 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1008,32 +1008,40 @@ class ZfsDataset: obsolete.destroy() self.snapshots.remove(obsolete) - def find_common_snapshot(self, target_dataset, guid_check): + def find_common_snapshot(self, target_dataset, guid_check, bookmark_name): """find latest common snapshot/bookmark between us and target returns None if its - an initial transfer. It preffers bookmarks over snapshots on the source side. Target side will always be a snapshots. + an initial transfer. + + On the source it prefers the specified bookmark_name + Args: :rtype: ZfsDataset|None :type guid_check: bool :type target_dataset: ZfsDataset + :type preferred_bookmark: str """ + bookmark = self.zfs_node.get_dataset(bookmark_name) + if not target_dataset.exists or not target_dataset.snapshots: # target has nothing yet return None else: for target_snapshot in reversed(target_dataset.snapshots): - # Source bookmark? - source_bookmark = self.find_bookmark(target_snapshot) - if source_bookmark: - if guid_check and source_bookmark.properties['guid'] != target_snapshot.properties['guid']: - source_bookmark.warning("Bookmark has mismatching GUID, ignoring.") - else: - source_bookmark.debug("Common bookmark") - return source_bookmark + # Source bookmark with same suffix? + # source_bookmark = self.find_bookmark(target_snapshot) + # if source_bookmark: + # if guid_check and source_bookmark.properties['guid'] != target_snapshot.properties['guid']: + # source_bookmark.warning("Bookmark has mismatching GUID, ignoring.") + # else: + # source_bookmark.debug("Common bookmark") + # return source_bookmark + if bookmark.exists and bookmark.properties['guid'] == target_snapshot.properties['guid']:XXX wil eigenlijk guid check opineel houden .dus bookmark name word snapshotname_targetdatasetguid + return bookmark - # Source snapshot? + # Source snapshot with same suffix? source_snapshot = self.find_snapshot(target_snapshot) if source_snapshot: if guid_check and source_snapshot.properties['guid'] != target_snapshot.properties['guid']: @@ -1138,7 +1146,7 @@ class ZfsDataset: else: return resume_token - def _plan_sync(self, target_dataset, also_other_snapshots, guid_check, raw): + def _plan_sync(self, target_dataset, also_other_snapshots, guid_check, raw, bookmark_name): """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. Args: @@ -1147,6 +1155,7 @@ class ZfsDataset: :type also_other_snapshots: bool :type guid_check: bool :type raw: bool + :type bookmark_name: str Returns: tuple: A tuple containing: @@ -1156,11 +1165,14 @@ class ZfsDataset: - list[ZfsDataset]: Transfer target snapshots. These need to be transferred. - list[ZfsDataset]: Incompatible target snapshots. Target snapshots that are in the way, after the common snapshot. (need to be destroyed to continue) + """ ### 1: determine common and start snapshot + target_dataset.debug("Determining start snapshot") - source_common_snapshot = self.find_common_snapshot(target_dataset, guid_check=guid_check) + source_common_snapshot = self.find_common_snapshot(target_dataset, guid_check=guid_check, + bookmark_name=bookmark_name) incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(source_common_snapshot, raw) # let thinner decide whats obsolete on source after the transfer is done @@ -1239,7 +1251,7 @@ class ZfsDataset: def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, - use_bookmarks, bookmark_tag): + use_bookmarks, bookmark_name): """sync this dataset's snapshots to target_dataset, while also thinning out old snapshots along the way. @@ -1259,7 +1271,7 @@ class ZfsDataset: :type no_send: bool :type guid_check: bool :type use_bookmarks: bool - :type bookmark_tag: str + :type bookmark_name: str """ # self.verbose("-> {}".format(target_dataset)) @@ -1282,7 +1294,7 @@ class ZfsDataset: (source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots) = \ self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, - guid_check=guid_check, raw=raw) + guid_check=guid_check, raw=raw, bookmark_name=bookmark_name) # NOTE: we do a pre-clean because we dont want filesystems to fillup when backups keep failing. # Also usefull with no_send to still cleanup stuff. @@ -1348,7 +1360,7 @@ class ZfsDataset: # bookmark common snapshot on source, or use holds if bookmarks are not enabled. if use_bookmarks: - source_bookmark = source_snapshot.bookmark(bookmark_tag) + source_bookmark = source_snapshot.bookmark(bookmark_name) # note: destroy source_snapshot when obsolete at this point? else: source_bookmark = None diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index be9b8b2..7d91c68 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -129,6 +129,7 @@ class ZfsNode(ExecuteNode): def get_dataset(self, name, force_exists=None): """get a ZfsDataset() object from name. stores objects internally to enable caching + :type name: str :rtype: ZfsDataset """ From 0ec493d231dfc19889a78054af378434429bd60e Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Sun, 6 Oct 2024 14:50:36 +0200 Subject: [PATCH 034/135] fix suffix stuff --- zfs_autobackup/ZfsAutobackup.py | 8 ++--- zfs_autobackup/ZfsDataset.py | 52 ++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index dbafadc..faed90b 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -357,9 +357,9 @@ class ZfsAutobackup(ZfsAuto): target_datasets[target_name] = source_dataset # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: - def sync_datasets(self, source_node, source_datasets, target_node, bookmark_name): + def sync_datasets(self, source_node, source_datasets, target_node, bookmark_tag): """Sync datasets, or thin-only on both sides - :type bookmark_name: str + :type bookmark_tag: str :type target_node: ZfsNode :type source_datasets: list of ZfsDataset :type source_node: ZfsNode @@ -421,7 +421,7 @@ class ZfsAutobackup(ZfsAuto): decrypt=self.args.decrypt, encrypt=self.args.encrypt, zfs_compressed=self.args.zfs_compressed, force=self.args.force, guid_check=not self.args.no_guid_check, use_bookmarks=use_bookmarks, - bookmark_name=bookmark_name) + bookmark_tag=bookmark_tag) except Exception as e: fail_count = fail_count + 1 @@ -559,7 +559,7 @@ class ZfsAutobackup(ZfsAuto): source_node=source_node, source_datasets=source_datasets, target_node=target_node, - bookmark_name=self.args.backup_name + self.tag_seperator + target_dataset.properties['guid']) + bookmark_tag=target_dataset.properties['guid']) # no target specified, run in snapshot-only mode else: diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 51f6a37..4e1b989 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -539,7 +539,7 @@ class ZfsDataset: """ for snapshot in snapshots: - if snapshot.suffix == self.suffix: + if snapshot.tagless_suffix == self.tagless_suffix: return snapshot return None @@ -557,23 +557,24 @@ class ZfsDataset: return None if not isinstance(snapshot, ZfsDataset): - suffix = snapshot + tagless_suffix = snapshot else: - suffix = snapshot.suffix + tagless_suffix = snapshot.tagless_suffix for snapshot in self.snapshots: - if snapshot.suffix == suffix: + if snapshot.tagless_suffix == tagless_suffix: return snapshot return None - def find_bookmark(self, bookmark): + def find_bookmark(self, bookmark, ignore_tag): """find bookmark by bookmark name (can be a suffix or a different ZfsDataset) Returns None if it cant find it. Args: :rtype: ZfsDataset|None :type bookmark: str|ZfsDataset|None + :type ignore_tag: bool """ if bookmark is None: @@ -582,10 +583,13 @@ class ZfsDataset: if not isinstance(bookmark, ZfsDataset): suffix = bookmark else: - suffix = bookmark.suffix + if ignore_tag: + suffix = bookmark.tagless_suffix + else: + suffix = bookmark.suffix for bookmark in self.bookmarks: - if bookmark.suffix == suffix: + if (ignore_tag and bookmark.tagless_suffix == suffix) or (not ignore_tag and bookmark.suffix == suffix): return bookmark return None @@ -601,11 +605,11 @@ class ZfsDataset: if not isinstance(snapshot, ZfsDataset): snapshot_name = snapshot else: - snapshot_name = snapshot.suffix + snapshot_name = snapshot.tagless_suffix index = 0 for snapshot in self.snapshots: - if snapshot.suffix == snapshot_name: + if snapshot.tagless_suffix == snapshot_name: return index index = index + 1 @@ -655,13 +659,14 @@ class ZfsDataset: self.debug("Bookmarking") + bookmark_name = "#" + self.tagless_suffix + self.zfs_node.tag_seperator + tag cmd = [ - "zfs", "bookmark", self.name, "#" + self.tagless_suffix + self.zfs_node.tag_seperator + tag + "zfs", "bookmark", self.name, bookmark_name ] self.zfs_node.run(cmd=cmd) - bookmark = self.zfs_node.get_dataset(self.name + '#' + self.suffix, force_exists=True) + bookmark = self.zfs_node.get_dataset(bookmark_name, force_exists=True) self.cache_snapshot_bookmark(bookmark) return bookmark @@ -1008,7 +1013,7 @@ class ZfsDataset: obsolete.destroy() self.snapshots.remove(obsolete) - def find_common_snapshot(self, target_dataset, guid_check, bookmark_name): + def find_common_snapshot(self, target_dataset, guid_check, bookmark_tag): """find latest common snapshot/bookmark between us and target returns None if its an initial transfer. @@ -1019,10 +1024,10 @@ class ZfsDataset: :rtype: ZfsDataset|None :type guid_check: bool :type target_dataset: ZfsDataset - :type preferred_bookmark: str + :type bookmark_tag: str """ - bookmark = self.zfs_node.get_dataset(bookmark_name) + bookmark = self.zfs_node.get_dataset(bookmark_tag) if not target_dataset.exists or not target_dataset.snapshots: # target has nothing yet @@ -1038,7 +1043,8 @@ class ZfsDataset: # else: # source_bookmark.debug("Common bookmark") # return source_bookmark - if bookmark.exists and bookmark.properties['guid'] == target_snapshot.properties['guid']:XXX wil eigenlijk guid check opineel houden .dus bookmark name word snapshotname_targetdatasetguid + if bookmark.exists and bookmark.properties['guid'] == target_snapshot.properties['guid']: + # FIXME: wil eigenlijk guid check opineel houden .dus bookmark name word snapshotname_targetdatasetguid return bookmark # Source snapshot with same suffix? @@ -1120,7 +1126,7 @@ class ZfsDataset: for target_snapshot in target_dataset.snapshots: if (target_snapshot in target_obsoletes) \ and (not source_common_snapshot or ( - target_snapshot.suffix != source_common_snapshot.suffix)): + target_snapshot.tagless_suffix != source_common_snapshot.tagless_suffix)): if target_snapshot.exists: target_snapshot.destroy() @@ -1146,7 +1152,7 @@ class ZfsDataset: else: return resume_token - def _plan_sync(self, target_dataset, also_other_snapshots, guid_check, raw, bookmark_name): + def _plan_sync(self, target_dataset, also_other_snapshots, guid_check, raw, bookmark_tag): """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. Args: @@ -1155,7 +1161,7 @@ class ZfsDataset: :type also_other_snapshots: bool :type guid_check: bool :type raw: bool - :type bookmark_name: str + :type bookmark_tag: str Returns: tuple: A tuple containing: @@ -1172,7 +1178,7 @@ class ZfsDataset: target_dataset.debug("Determining start snapshot") source_common_snapshot = self.find_common_snapshot(target_dataset, guid_check=guid_check, - bookmark_name=bookmark_name) + bookmark_tag=bookmark_tag) incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(source_common_snapshot, raw) # let thinner decide whats obsolete on source after the transfer is done @@ -1251,7 +1257,7 @@ class ZfsDataset: def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, - use_bookmarks, bookmark_name): + use_bookmarks, bookmark_tag): """sync this dataset's snapshots to target_dataset, while also thinning out old snapshots along the way. @@ -1271,7 +1277,7 @@ class ZfsDataset: :type no_send: bool :type guid_check: bool :type use_bookmarks: bool - :type bookmark_name: str + :type bookmark_tag: str """ # self.verbose("-> {}".format(target_dataset)) @@ -1294,7 +1300,7 @@ class ZfsDataset: (source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots) = \ self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, - guid_check=guid_check, raw=raw, bookmark_name=bookmark_name) + guid_check=guid_check, raw=raw, bookmark_tag=bookmark_tag) # NOTE: we do a pre-clean because we dont want filesystems to fillup when backups keep failing. # Also usefull with no_send to still cleanup stuff. @@ -1360,7 +1366,7 @@ class ZfsDataset: # bookmark common snapshot on source, or use holds if bookmarks are not enabled. if use_bookmarks: - source_bookmark = source_snapshot.bookmark(bookmark_name) + source_bookmark = source_snapshot.bookmark(bookmark_tag) # note: destroy source_snapshot when obsolete at this point? else: source_bookmark = None From 231a3908183424043b2e738cccdb11d15762c9e3 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Sun, 6 Oct 2024 23:04:04 +0200 Subject: [PATCH 035/135] wip fixes --- tests/test_zfsautobackup34.py | 2 +- zfs_autobackup/ZfsAutobackup.py | 2 +- zfs_autobackup/ZfsDataset.py | 24 ++++++++++++++++++------ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/test_zfsautobackup34.py b/tests/test_zfsautobackup34.py index 8e64d8b..6017ba4 100644 --- a/tests/test_zfsautobackup34.py +++ b/tests/test_zfsautobackup34.py @@ -190,7 +190,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 with mocktime("20101111000001"): self.assertFalse( ZfsAutobackup( - "test test_target1/a --no-progress --verbose --allow-empty --tag tag1".split(" ")).run()) + "test test_target1/a --no-progress --verbose --allow-empty --debug --tag tag1".split(" ")).run()) # increment, should be from bookmark with mocktime("20101111000002"): diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index faed90b..0b4999f 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -224,7 +224,7 @@ class ZfsAutobackup(ZfsAuto): # does it have other snapshots? has_others = False for snapshot in dataset.snapshots: - if not snapshot.is_ours(): + if not snapshot.is_ours: has_others = True break diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 4e1b989..2213362 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -62,6 +62,8 @@ class ZfsDataset: return self.name def __eq__(self, dataset): + """compare the full name of the dataset""" + if not isinstance(dataset, ZfsDataset): return False @@ -145,10 +147,10 @@ class ZfsDataset: @property def tagless_suffix(self): - """snapshot or bookmark part of the name, but without the tag.""" + """snapshot or bookmark part of the name, but without the tag. (if its our snapshot we remove the tag)""" suffix = self.suffix - if self.zfs_node.tag_seperator in suffix: + if self.is_ours and self.zfs_node.tag_seperator in suffix: return suffix.split(self.zfs_node.tag_seperator)[0] else: return suffix @@ -286,6 +288,8 @@ class ZfsDataset: def find_next_snapshot(self, snapshot_bookmark): """find next snapshot in this dataset, according to snapshot or bookmark. None if it doesn't exist + + Args: :type snapshot_bookmark: ZfsDataset """ @@ -298,7 +302,7 @@ class ZfsDataset: if snapshot == snapshot_bookmark: found = True else: - if found == True and snapshot.is_snapshot: + if found and snapshot.is_snapshot: return snapshot return None @@ -417,6 +421,7 @@ class ZfsDataset: else: return True + @property def is_ours(self): """return true if this snapshot name belong to the current backup_name and snapshot formatting""" return self.timestamp is not None @@ -454,8 +459,13 @@ class ZfsDataset: :rtype: int|None """ + if self.zfs_node.tag_seperator in self.suffix: + tagless_suffix = self.suffix.split(self.zfs_node.tag_seperator)[0] + else: + tagless_suffix = self.suffix + try: - dt = datetime.strptime(self.tagless_suffix, self.zfs_node.snapshot_time_format) + dt = datetime.strptime(tagless_suffix, self.zfs_node.snapshot_time_format) except ValueError: return None @@ -526,7 +536,7 @@ class ZfsDataset: ret = [] for snapshot in self.snapshots: - if snapshot.is_ours(): + if snapshot.is_ours: ret.append(snapshot) return ret @@ -548,6 +558,8 @@ class ZfsDataset: """find snapshot by snapshot name (can be a suffix or a different ZfsDataset) Returns None if it cant find it. + Note that matches with our own snapshots will be done tagless. + Args: :rtype: ZfsDataset|None :type snapshot: str|ZfsDataset|None @@ -1206,7 +1218,7 @@ class ZfsDataset: while source_snapshot: # we want it? - if (also_other_snapshots or source_snapshot.is_ours()) and not source_snapshot.is_snapshot_excluded: + if (also_other_snapshots or source_snapshot.is_ours) and not source_snapshot.is_snapshot_excluded: # create virtual target snapshot target_snapshot = target_dataset.zfs_node.get_dataset( target_dataset.filesystem_name + source_snapshot.typed_suffix, force_exists=False) From 4e7679cf2f72c4b6c92df8eb3a47bd503245b728 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Sun, 6 Oct 2024 23:36:15 +0200 Subject: [PATCH 036/135] preferred bookmark matching --- zfs_autobackup/ZfsDataset.py | 55 +++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 2213362..f452777 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -168,6 +168,15 @@ class ZfsDataset: raise (Exception("This is not a snapshot or bookmark")) + @property + def tag(self): + """the tag-part of a snapshot or bookmark, if any. can return None""" + + if self.is_ours and self.zfs_node.tag_seperator in self.suffix: + return self.suffix.split(self.zfs_node.tag_seperator)[1] + else: + return None + @property def is_snapshot(self): """true if this dataset is a snapshot""" @@ -579,30 +588,33 @@ class ZfsDataset: return None - def find_bookmark(self, bookmark, ignore_tag): + def find_bookmark(self, snapshot_bookmark, preferred_tag): """find bookmark by bookmark name (can be a suffix or a different ZfsDataset) Returns None if it cant find it. + We try to find the bookmark with the preferred tag (which is usually a target path guid, to prevent conflicting bookmarks by multiple sends) + Args: :rtype: ZfsDataset|None - :type bookmark: str|ZfsDataset|None + :type snapshot_bookmark: str|ZfsDataset|None :type ignore_tag: bool """ - if bookmark is None: + if snapshot_bookmark is None: return None - if not isinstance(bookmark, ZfsDataset): - suffix = bookmark + if not isinstance(snapshot_bookmark, ZfsDataset): + tagless_suffix = snapshot_bookmark else: - if ignore_tag: - suffix = bookmark.tagless_suffix - else: - suffix = bookmark.suffix + tagless_suffix = snapshot_bookmark.tagless_suffix - for bookmark in self.bookmarks: - if (ignore_tag and bookmark.tagless_suffix == suffix) or (not ignore_tag and bookmark.suffix == suffix): - return bookmark + for snapshot_bookmark in self.bookmarks: + if snapshot_bookmark.tagless_suffix == tagless_suffix and snapshot_bookmark.tag == preferred_tag: + return snapshot_bookmark + + for snapshot_bookmark in self.bookmarks: + if snapshot_bookmark.tagless_suffix == tagless_suffix: + return snapshot_bookmark return None @@ -1047,17 +1059,14 @@ class ZfsDataset: else: for target_snapshot in reversed(target_dataset.snapshots): - # Source bookmark with same suffix? - # source_bookmark = self.find_bookmark(target_snapshot) - # if source_bookmark: - # if guid_check and source_bookmark.properties['guid'] != target_snapshot.properties['guid']: - # source_bookmark.warning("Bookmark has mismatching GUID, ignoring.") - # else: - # source_bookmark.debug("Common bookmark") - # return source_bookmark - if bookmark.exists and bookmark.properties['guid'] == target_snapshot.properties['guid']: - # FIXME: wil eigenlijk guid check opineel houden .dus bookmark name word snapshotname_targetdatasetguid - return bookmark + # Prefer bookmarks over snapshots + source_bookmark = self.find_bookmark(target_snapshot, preferred_tag=bookmark_tag) + if source_bookmark: + if guid_check and source_bookmark.properties['guid'] != target_snapshot.properties['guid']: + source_bookmark.warning("Bookmark has mismatching GUID, ignoring.") + else: + source_bookmark.debug("Common bookmark") + return source_bookmark # Source snapshot with same suffix? source_snapshot = self.find_snapshot(target_snapshot) From 49069dd3c9e7fc06d747d07d57745506b5ebbe34 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Mon, 7 Oct 2024 15:24:35 +0200 Subject: [PATCH 037/135] bookmarks now use target dataset guid as tag, to allow sending to multiple destinations --- tests/test_zfsautobackup34.py | 62 ++++++++++++++++++++++++++++++++--- zfs_autobackup/ZfsDataset.py | 26 ++++++++------- 2 files changed, 72 insertions(+), 16 deletions(-) diff --git a/tests/test_zfsautobackup34.py b/tests/test_zfsautobackup34.py index 6017ba4..513a34f 100644 --- a/tests/test_zfsautobackup34.py +++ b/tests/test_zfsautobackup34.py @@ -190,25 +190,77 @@ test_target1/test_source2/fs2/sub@test-20101111000003 with mocktime("20101111000001"): self.assertFalse( ZfsAutobackup( - "test test_target1/a --no-progress --verbose --allow-empty --debug --tag tag1".split(" ")).run()) + "test test_target1/a --no-progress --verbose --allow-empty --debug --tag tagA".split(" ")).run()) # increment, should be from bookmark with mocktime("20101111000002"): self.assertFalse( ZfsAutobackup( - "test test_target1/a --no-progress --verbose --allow-empty --tag tag1".split(" ")).run()) + "test test_target1/a --no-progress --verbose --allow-empty --tag tagB".split(" ")).run()) # to target b, now each has one full + two incrementals, which should be from their own bookmarks. with mocktime("20101111000003"): self.assertFalse( ZfsAutobackup( - "test test_target1/b --no-progress --verbose --no-snapshot --allow-empty".split( + "test test_target1/b --no-progress --verbose --allow-empty".split( " ")).run()) # result: # for target a the bookmarks should be at 20101111000002, for target b the bookmarks should be at 20101111000003 r = shelltest("zfs list -H -r -t all -o name " + TEST_POOLS) - self.assertMultiLineEqual(r, """ -... + self.assertRegexpMatches(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000001_tagA +test_source1/fs1@test-20101111000002_tagB +test_source1/fs1@test-20101111000003 +test_source1/fs1#test-20101111000002_[0-9]* +test_source1/fs1#test-20101111000003_[0-9]* +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000001_tagA +test_source1/fs1/sub@test-20101111000002_tagB +test_source1/fs1/sub@test-20101111000003 +test_source1/fs1/sub#test-20101111000002_[0-9]* +test_source1/fs1/sub#test-20101111000003_[0-9]* +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000001_tagA +test_source2/fs2/sub@test-20101111000002_tagB +test_source2/fs2/sub@test-20101111000003 +test_source2/fs2/sub#test-20101111000002_[0-9]* +test_source2/fs2/sub#test-20101111000003_[0-9]* +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/a +test_target1/a/test_source1 +test_target1/a/test_source1/fs1 +test_target1/a/test_source1/fs1@test-20101111000001_tagA +test_target1/a/test_source1/fs1@test-20101111000002_tagB +test_target1/a/test_source1/fs1/sub +test_target1/a/test_source1/fs1/sub@test-20101111000001_tagA +test_target1/a/test_source1/fs1/sub@test-20101111000002_tagB +test_target1/a/test_source2 +test_target1/a/test_source2/fs2 +test_target1/a/test_source2/fs2/sub +test_target1/a/test_source2/fs2/sub@test-20101111000001_tagA +test_target1/a/test_source2/fs2/sub@test-20101111000002_tagB +test_target1/b +test_target1/b/test_source1 +test_target1/b/test_source1/fs1 +test_target1/b/test_source1/fs1@test-20101111000001_tagA +test_target1/b/test_source1/fs1@test-20101111000002_tagB +test_target1/b/test_source1/fs1@test-20101111000003 +test_target1/b/test_source1/fs1/sub +test_target1/b/test_source1/fs1/sub@test-20101111000001_tagA +test_target1/b/test_source1/fs1/sub@test-20101111000002_tagB +test_target1/b/test_source1/fs1/sub@test-20101111000003 +test_target1/b/test_source2 +test_target1/b/test_source2/fs2 +test_target1/b/test_source2/fs2/sub +test_target1/b/test_source2/fs2/sub@test-20101111000001_tagA +test_target1/b/test_source2/fs2/sub@test-20101111000002_tagB +test_target1/b/test_source2/fs2/sub@test-20101111000003 """) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index f452777..bd135e2 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -12,6 +12,10 @@ class ZfsDataset: """a zfs dataset (filesystem/volume/snapshot/clone) Note that a dataset doesn't have to actually exist (yet/anymore) Also most properties are cached for performance-reasons, but also to allow --test to function correctly. + + A dataset 'name' is the full name as you would use it with zfs list. + Internally we use it in 3 parts: the prefix, suffix and tag. + (see respective properties below) """ # illegal properties per dataset type. these will be removed from --set-properties and --filter-properties @@ -122,7 +126,7 @@ class ZfsDataset: return "/".join(self.split_path()[:-count]) @property - def filesystem_name(self): + def prefix(self): """filesystem part of the name (before the @)""" if self.is_snapshot: (filesystem, snapshot) = self.name.split("@") @@ -286,7 +290,7 @@ class ZfsDataset: :rtype: ZfsDataset | None """ if self.is_snapshot: - return self.zfs_node.get_dataset(self.filesystem_name) + return self.zfs_node.get_dataset(self.prefix) else: stripped = self.rstrip_path(1) if stripped: @@ -683,7 +687,7 @@ class ZfsDataset: self.debug("Bookmarking") - bookmark_name = "#" + self.tagless_suffix + self.zfs_node.tag_seperator + tag + bookmark_name = self.prefix + "#" + self.tagless_suffix + self.zfs_node.tag_seperator + tag cmd = [ "zfs", "bookmark", self.name, bookmark_name ] @@ -802,7 +806,6 @@ class ZfsDataset: cmd.extend(send_pipes) - self.error(cmd) output_pipe = self.zfs_node.run(cmd, pipe=True, readonly=True) return output_pipe @@ -857,7 +860,7 @@ class ZfsDataset: self.debug("Enabled resume support") cmd.append("-s") - cmd.append(self.filesystem_name) + cmd.append(self.prefix) if ignore_exit_code: valid_exitcodes = [] @@ -935,17 +938,17 @@ class ZfsDataset: if filter_properties is None: filter_properties = [] - self.debug("Transfer snapshot to {}".format(target_snapshot.filesystem_name)) + self.debug("Transfer snapshot to {}".format(target_snapshot.prefix)) if resume_token: self.verbose("resuming") # initial or increment if not prev_snapshot: - self.verbose("-> {} (new)".format(target_snapshot.filesystem_name)) + self.verbose("-> {} (new)".format(target_snapshot.prefix)) else: # incremental - self.verbose("-> {}".format(target_snapshot.filesystem_name)) + self.verbose("-> {}".format(target_snapshot.prefix)) # do it pipe = self.send_pipe(features=features, show_progress=show_progress, prev_snapshot=prev_snapshot, @@ -994,7 +997,7 @@ class ZfsDataset: matches = re.findall("toname = .*@(.*)", line) if matches: snapshot_name = matches[0] - snapshot = self.zfs_node.get_dataset(self.filesystem_name + "@" + snapshot_name) + snapshot = self.zfs_node.get_dataset(self.prefix + "@" + snapshot_name) snapshot.debug("resume token belongs to this snapshot") return snapshot @@ -1230,7 +1233,7 @@ class ZfsDataset: if (also_other_snapshots or source_snapshot.is_ours) and not source_snapshot.is_snapshot_excluded: # create virtual target snapshot target_snapshot = target_dataset.zfs_node.get_dataset( - target_dataset.filesystem_name + source_snapshot.typed_suffix, force_exists=False) + target_dataset.prefix + source_snapshot.typed_suffix, force_exists=False) possible_target_snapshots.append(target_snapshot) source_snapshot = self.find_next_snapshot(source_snapshot) @@ -1401,7 +1404,8 @@ class ZfsDataset: # FIXME: met bookmarks kan de huidige snapshot na send ook meteen weg # FIXME: klopt niet, nu haalt ie altijd bookmark weg? wat als we naar andere target willen senden (zoals in test_encryption.py) if prev_source_snapshot_bookmark and ( - prev_source_snapshot_bookmark in source_obsoletes or prev_source_snapshot_bookmark.is_bookmark): + prev_source_snapshot_bookmark in source_obsoletes or ( + prev_source_snapshot_bookmark.is_bookmark and prev_source_snapshot_bookmark.tag == bookmark_tag)): prev_source_snapshot_bookmark.destroy() # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, From 97123502b058efb8ecbfc3452b683dfedc5e0eaf Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Mon, 7 Oct 2024 15:26:57 +0200 Subject: [PATCH 038/135] devwarning --- README.md | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 57bea6e..022178e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ - # ZFS autobackup +# NOTE: this is the v3.4 development branch. Not ready for testing yet! + [![Tests](https://github.com/psy0rz/zfs_autobackup/workflows/Regression%20tests/badge.svg)](https://github.com/psy0rz/zfs_autobackup/actions?query=workflow%3A%22Regression+tests%22) [![Coverage Status](https://coveralls.io/repos/github/psy0rz/zfs_autobackup/badge.svg)](https://coveralls.io/github/psy0rz/zfs_autobackup) [![Python Package](https://github.com/psy0rz/zfs_autobackup/workflows/Upload%20Python%20Package/badge.svg)](https://pypi.org/project/zfs-autobackup/) [![CodeQL](https://github.com/psy0rz/zfs_autobackup/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/psy0rz/zfs_autobackup/actions/workflows/codeql-analysis.yml) @@ -10,26 +11,31 @@ ZFS-autobackup tries to be the most reliable and easiest to use tool, while havi You can either use it as a **backup** tool, **replication** tool or **snapshot** tool. -You can select what to backup by setting a custom `ZFS property`. This makes it easy to add/remove specific datasets, or just backup your whole pool. +You can select what to backup by setting a custom `ZFS property`. This makes it easy to add/remove specific datasets, or +just backup your whole pool. -Other settings are just specified on the commandline: Simply setup and test your zfs-autobackup command and fix all the issues you might encounter. When you're done you can just copy/paste your command to a cron or script. +Other settings are just specified on the commandline: Simply setup and test your zfs-autobackup command and fix all the +issues you might encounter. When you're done you can just copy/paste your command to a cron or script. -Since it's using ZFS commands, you can see what it's actually doing by specifying `--debug`. This also helps a lot if you run into some strange problem or errors. You can just copy-paste the command that fails and play around with it on the commandline. (something I missed in other tools) +Since it's using ZFS commands, you can see what it's actually doing by specifying `--debug`. This also helps a lot if +you run into some strange problem or errors. You can just copy-paste the command that fails and play around with it on +the commandline. (something I missed in other tools) -An important feature that's missing from other tools is a reliable `--test` option: This allows you to see what zfs-autobackup will do and tune your parameters. It will do everything, except make changes to your system. +An important feature that's missing from other tools is a reliable `--test` option: This allows you to see what +zfs-autobackup will do and tune your parameters. It will do everything, except make changes to your system. ## Features * Works across operating systems: Tested with **Linux**, **FreeBSD/FreeNAS** and **SmartOS**. -* Low learning curve: no complex daemons or services, no additional software or networking needed. +* Low learning curve: no complex daemons or services, no additional software or networking needed. * Plays nicely with existing replication systems. (Like Proxmox HA) -* Automatically selects filesystems to backup by looking at a simple ZFS property. +* Automatically selects filesystems to backup by looking at a simple ZFS property. * Creates consistent snapshots. (takes all snapshots at once, atomicly.) * Multiple backups modes: - * Backup local data on the same server. - * "push" local data to a backup-server via SSH. - * "pull" remote data from a server via SSH and backup it locally. - * "pull+push": Zero trust between source and target. + * Backup local data on the same server. + * "push" local data to a backup-server via SSH. + * "pull" remote data from a server via SSH and backup it locally. + * "pull+push": Zero trust between source and target. * Can be scheduled via simple cronjob or run directly from commandline. * Also supports complex backup geometries. * ZFS encryption support: Can decrypt / encrypt or even re-encrypt datasets during transfer. @@ -48,10 +54,10 @@ An important feature that's missing from other tools is a reliable `--test` opti * Complete and clean logging. * All code is regression tested against actual ZFS environments. * Easy installation: - * Just install zfs-autobackup via pip. - * Only needs to be installed on one side. - * Written in python and uses zfs-commands, no special 3rd party dependency's or compiled libraries needed. - * No annoying config files or properties. + * Just install zfs-autobackup via pip. + * Only needs to be installed on one side. + * Written in python and uses zfs-commands, no special 3rd party dependency's or compiled libraries needed. + * No annoying config files or properties. ## Getting started @@ -63,6 +69,6 @@ Or read the [Full manual](https://github.com/psy0rz/zfs_autobackup/wiki/Manual) This project was sponsored by: -* JetBrains +* JetBrains * https://rsync.net * [DatuX](https://www.datux.nl) From d97a5f952741f81f23e725ba977ebe21014da0a7 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 8 Oct 2024 16:51:03 +0200 Subject: [PATCH 039/135] fix --- tests/test_encryption.py | 160 +++++++++++++++++++---------------- zfs_autobackup/ZfsDataset.py | 3 +- 2 files changed, 91 insertions(+), 72 deletions(-) diff --git a/tests/test_encryption.py b/tests/test_encryption.py index 257ade6..3193f26 100644 --- a/tests/test_encryption.py +++ b/tests/test_encryption.py @@ -2,6 +2,7 @@ from zfs_autobackup.CmdPipe import CmdPipe from basetest import * import time + # We have to do a LOT to properly test encryption/decryption/raw transfers # # For every scenario we need at least: @@ -20,7 +21,6 @@ import time class TestZfsEncryption(unittest2.TestCase): - def setUp(self): prepare_zpools() @@ -40,7 +40,9 @@ class TestZfsEncryption(unittest2.TestCase): # create encrypted source dataset shelltest("rm /tmp/zfstest.key 2>/dev/null;true") shelltest("echo {} > /tmp/zfstest.key".format(key)) - shelltest("zfs create -o keylocation=file:///tmp/zfstest.key -o keyformat=passphrase -o encryption=on {}".format(path)) + shelltest( + "zfs create -o keylocation=file:///tmp/zfstest.key -o keyformat=passphrase -o encryption=on {}".format( + path)) if unload_key: shelltest("zfs unmount {}".format(path)) @@ -48,23 +50,30 @@ class TestZfsEncryption(unittest2.TestCase): # r=shelltest("dd if=/dev/zero of=/test_source1/fs1/enc1/data.txt bs=200000 count=1") - def test_raw(self): + def test_raw(self): """send encrypted data unaltered (standard operation)""" self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") - self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsourcekeyless", unload_key=True) # raw mode shouldn't need a key + self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsourcekeyless", + unload_key=True) # raw mode shouldn't need a key self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split( + " ")).run()) with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split( + " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") - self.assertMultiLineEqual(r,""" + self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_target1 encryptionroot - - test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget - @@ -86,22 +95,28 @@ test_target1/test_source2/fs2 encryption test_target1/test_source2/fs2/sub encryptionroot - - """) - def test_decrypt(self): + def test_decrypt(self): """decrypt data and store unencrypted (--decrypt)""" - - self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split( + " ")).run()) # NOTE: this also tests if sending a source to a second target works correctly (this failed during bookmark implementation) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received --debug".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received --debug".split( + " ")).run()) with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split( + " ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received".split( + " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") self.assertEqual(r, """ @@ -124,19 +139,27 @@ test_target1/test_source2/fs2 encryptionroot - test_target1/test_source2/fs2/sub encryptionroot - - """) - def test_encrypt(self): + def test_encrypt(self): """send normal data set and store encrypted on the other side (--encrypt) issue #60 """ self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split( + " ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split( + " ")).run()) with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split( + " ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split( + " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") self.assertEqual(r, """ @@ -167,14 +190,16 @@ test_target1/test_source2/fs2/sub encryptionroot - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) + "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split( + " ")).run()) self.assertFalse(ZfsAutobackup( "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split( " ")).run()) with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received".split(" ")).run()) + "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received".split( + " ")).run()) self.assertFalse(ZfsAutobackup( "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --exclude-received".split( " ")).run()) @@ -200,10 +225,7 @@ test_target1/test_source2/fs2 encryptionroot - test_target1/test_source2/fs2/sub encryptionroot - - """) - - - - def test_raw_invalid_snapshot(self): + def test_raw_invalid_snapshot(self): """in raw mode, its not allowed to have any newer snaphots on target, #219""" self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") @@ -211,18 +233,19 @@ test_target1/test_source2/fs2/sub encryptionroot - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress".split(" ")).run()) - #this is invalid in raw mode + # this is invalid in raw mode shelltest("zfs snapshot test_target1/test_source1/fs1/encryptedsource@incompatible") with mocktime("20101111000001"): - #should fail because of incompatble snapshot - self.assertEqual(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty".split(" ")).run(),1) - #should destroy incompatible and continue - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --no-snapshot --destroy-incompatible".split(" ")).run()) - + # should fail because of incompatble snapshot + self.assertEqual(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty".split(" ")).run(), + 1) + # should destroy incompatible and continue + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --no-snapshot --destroy-incompatible".split(" ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") - self.assertMultiLineEqual(r,""" + self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_target1 encryptionroot - - test_target1/test_source1 encryptionroot - - @@ -234,16 +257,16 @@ test_target1/test_source2/fs2 encryptionroot - test_target1/test_source2/fs2/sub encryptionroot - - """) - - def test_resume_encrypt_with_no_key(self): + def test_resume_encrypt_with_no_key(self): """test what happens if target encryption key not loaded (this led to a kernel crash of freebsd with 2.1.x i think) while trying to resume""" self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") - with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --exclude-received --clear-mountpoint".split( + " ")).run()) r = shelltest("zfs set compress=off test_source1 test_target1") @@ -274,43 +297,38 @@ test_target1/test_source2/fs2/sub encryptionroot - with mocktime("20101111000001"): self.assertEqual(ZfsAutobackup( "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint".split( - " ")).run(),3) + " ")).run(), 3) + # NOTE: On some versions this leaves 2 weird sub-datasets that should'nt be there (its probably a zfs bug?) + # so we ignore this, and just make sure the backup resumes correctly after reloading the key. + # r = shelltest("zfs get -r -t all encryptionroot test_target1") + # self.assertEqual(r, """ + # NAME PROPERTY VALUE SOURCE + # test_target1 encryptionroot - - + # test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1@test-20101111000000 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - + # test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000000 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - + # test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000001 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - + # test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1/sub/sub encryptionroot - - + # test_target1/encryptedtarget/test_source1/fs1/sub/sub@test-20101111000001 encryptionroot - - + # test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source2/fs2/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source2/fs2/sub/sub encryptionroot - - + # test_target1/encryptedtarget/test_source2/fs2/sub/sub@test-20101111000001 encryptionroot - - + # """) - -#NOTE: On some versions this leaves 2 weird sub-datasets that should'nt be there (its probably a zfs bug?) -#so we ignore this, and just make sure the backup resumes correctly after reloading the key. -# r = shelltest("zfs get -r -t all encryptionroot test_target1") -# self.assertEqual(r, """ -# NAME PROPERTY VALUE SOURCE -# test_target1 encryptionroot - - -# test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1@test-20101111000000 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - -# test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000000 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - -# test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000001 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - -# test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1/sub/sub encryptionroot - - -# test_target1/encryptedtarget/test_source1/fs1/sub/sub@test-20101111000001 encryptionroot - - -# test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source2/fs2/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source2/fs2/sub/sub encryptionroot - - -# test_target1/encryptedtarget/test_source2/fs2/sub/sub@test-20101111000001 encryptionroot - - -# """) - - - - #reload key and resume correctly. + # reload key and resume correctly. self.load_key("22222222", "test_target1/encryptedtarget") # resume should complete with mocktime("20101111000001"): self.assertEqual(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint".split( - " ")).run(),0) - + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint --debug".split( + " ")).run(), 0) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index bd135e2..fc6e3c7 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -984,6 +984,7 @@ class ZfsDataset: Args: :type resume_token: str + :rtype: ZfsDataset|None """ # use zfs send -n option to determine this # NOTE: on smartos stderr, on linux stdout @@ -1170,7 +1171,7 @@ class ZfsDataset: resume_token = target_dataset.properties['receive_resume_token'] # not valid anymore resume_snapshot = self.get_resume_snapshot(resume_token) - if not resume_snapshot or start_snapshot.suffix != resume_snapshot.snapshot_name: + if not resume_snapshot or start_snapshot.suffix != resume_snapshot.suffix: target_dataset.verbose("Aborting resume, its no longer valid.") target_dataset.abort_resume() else: From d79d6af54884bc127bf1c421de7cb21c53037ef8 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 8 Oct 2024 17:08:55 +0200 Subject: [PATCH 040/135] fixes --- tests/test_externalfailures.py | 23 +++++++---------------- tests/test_regressions.py | 11 +++-------- tests/test_zfsautobackup34.py | 12 ++++++++++++ 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/tests/test_externalfailures.py b/tests/test_externalfailures.py index 05f25a0..c14bb97 100644 --- a/tests/test_externalfailures.py +++ b/tests/test_externalfailures.py @@ -10,7 +10,6 @@ class TestExternalFailures(unittest2.TestCase): # generate a resumable state # NOTE: this generates two resumable test_target1/test_source1/fs1 and test_target1/test_source1/fs1/sub def generate_resume(self): - r = shelltest("zfs set compress=off test_source1 test_target1") # big change on source @@ -30,7 +29,6 @@ class TestExternalFailures(unittest2.TestCase): # resume initial backup def test_initial_resume(self): - # inital backup, leaves resume token with mocktime("20101111000000"): self.generate_resume() @@ -69,7 +67,6 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # resume incremental backup def test_incremental_resume(self): - # initial backup with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) @@ -114,8 +111,6 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # generate an invalid resume token, and verify if its aborted automaticly def test_initial_resumeabort(self): - - # inital backup, leaves resume token with mocktime("20101111000000"): self.generate_resume() @@ -150,7 +145,6 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # generate an invalid resume token, and verify if its aborted automaticly def test_incremental_resumeabort(self): - # initial backup with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) @@ -187,7 +181,6 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # create a resume situation, where the other side doesnt want the snapshot anymore ( should abort resume ) def test_abort_unwanted_resume(self): - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) @@ -200,7 +193,8 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # incremental, doesnt want previous anymore with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --keep-target=0 --allow-empty --debug".split(" ")).run()) + "test test_target1 --no-progress --verbose --keep-target=0 --allow-empty --debug".split( + " ")).run()) print(buf.getvalue()) @@ -222,7 +216,6 @@ test_target1/test_source2/fs2/sub@test-20101111000002 # test with empty snapshot list (this was a bug) def test_abort_resume_emptysnapshotlist(self): - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) @@ -240,15 +233,14 @@ test_target1/test_source2/fs2/sub@test-20101111000002 "test test_target1 --no-progress --verbose --no-snapshot".split( " ")).run()) - #NOTE: v3.4 doesnt abort the resume if there are no snapshots to send. + # NOTE: v3.4 doesnt abort the resume if there are no snapshots to send. # print(buf.getvalue()) # self.assertIn("Aborting resume, its obsolete", buf.getvalue()) - def test_missing_common(self): - with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --no-bookmarks".split(" ")).run()) # remove common snapshot and leave nothing shelltest("zfs release zfs_autobackup:test test_source1/fs1@test-20101111000000") @@ -257,10 +249,9 @@ test_target1/test_source2/fs2/sub@test-20101111000002 with mocktime("20101111000001"): self.assertTrue(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - #UPDATE: offcourse the one thing that wasn't tested had a bug :( (in ExecuteNode.run()). + # UPDATE: offcourse the one thing that wasn't tested had a bug :( (in ExecuteNode.run()). def test_ignoretransfererrors(self): - - self.skipTest("Not sure how to implement a test for this without some serious hacking and patching.") + self.skipTest("Not sure how to implement a test for this without some serious hacking and patching.") # #recreate target pool without any features # # shelltest("zfs set compress=on test_source1; zpool destroy test_target1; zpool create test_target1 -o feature@project_quota=disabled /dev/ram2") diff --git a/tests/test_regressions.py b/tests/test_regressions.py index 7a5dac9..273636f 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -1,4 +1,3 @@ - from basetest import * @@ -6,7 +5,7 @@ class TestZfsNode(unittest2.TestCase): def setUp(self): prepare_zpools() - self.longMessage=True + self.longMessage = True def test_keepsource0target10queuedsend(self): """Test if thinner doesnt destroy too much early on if there are no common snapshots YET. Issue #84""" @@ -26,7 +25,7 @@ class TestZfsNode(unittest2.TestCase): "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty".split( " ")).run()) - r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t bookmark,filesystem " + TEST_POOLS) self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 @@ -57,7 +56,6 @@ test_target1/test_source2/fs2/sub@test-20101111000001 test_target1/test_source2/fs2/sub@test-20101111000002 """) - def test_excludepaths(self): """Test issue #103""" @@ -70,8 +68,7 @@ test_target1/test_source2/fs2/sub@test-20101111000002 "test test_target1/target --no-progress --verbose --allow-empty".split( " ")).run()) - - r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 @@ -101,5 +98,3 @@ test_target1/target/test_target1/target_shouldnotbeexcluded@test-20101111000000 test_target1/target_shouldnotbeexcluded test_target1/target_shouldnotbeexcluded@test-20101111000000 """) - - diff --git a/tests/test_zfsautobackup34.py b/tests/test_zfsautobackup34.py index 513a34f..ea7943b 100644 --- a/tests/test_zfsautobackup34.py +++ b/tests/test_zfsautobackup34.py @@ -264,3 +264,15 @@ test_target1/b/test_source2/fs2/sub@test-20101111000001_tagA test_target1/b/test_source2/fs2/sub@test-20101111000002_tagB test_target1/b/test_source2/fs2/sub@test-20101111000003 """) + + def test_missing_common_bookmark(self): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --no-holds".split(" ")).run()) + + # remove common bookmark + bookmark = shelltest("zfs list -H -o name -t bookmark test_source1/fs1").strip() + shelltest("zfs destroy " + bookmark) + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) From cf541c581c2d2329aa6d9642fce3f1fed7c0aa5e Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 8 Oct 2024 17:13:30 +0200 Subject: [PATCH 041/135] dont list bookmarks in existing tests --- tests/test_destroymissing.py | 87 ++++---- tests/test_encryption.py | 2 +- tests/test_externalfailures.py | 12 +- tests/test_regressions.py | 2 +- tests/test_sendrecvpipes.py | 10 +- tests/test_zfsautobackup.py | 349 +++++++++++++++++---------------- tests/test_zfsautobackup31.py | 40 ++-- tests/test_zfsautobackup32.py | 85 ++++---- tests/test_zfsautobackup34.py | 10 +- tests/test_zfscheck.py | 69 ++++--- tests/test_zfsnode.py | 79 +++++--- 11 files changed, 386 insertions(+), 359 deletions(-) diff --git a/tests/test_destroymissing.py b/tests/test_destroymissing.py index 89eb4ec..8e9edb2 100644 --- a/tests/test_destroymissing.py +++ b/tests/test_destroymissing.py @@ -1,4 +1,3 @@ - from basetest import * @@ -6,125 +5,127 @@ class TestZfsNode(unittest2.TestCase): def setUp(self): prepare_zpools() - self.longMessage=True + self.longMessage = True - - - def test_destroymissing(self): - - #initial backup - with mocktime("19101111000000"): #1000 years in past + def test_destroymissing(self): + # initial backup + with mocktime("19101111000000"): # 1000 years in past self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds".split(" ")).run()) - with mocktime("20101111000000"): #far in past - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) - + with mocktime("20101111000000"): # far in past + self.assertFalse( + ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) with self.subTest("Should do nothing yet"): with OutputIO() as buf: with redirect_stdout(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split( + " ")).run()) print(buf.getvalue()) self.assertNotIn(": Destroy missing", buf.getvalue()) - with self.subTest("missing dataset of us that still has children"): - - #just deselect it so it counts as 'missing' + # just deselect it so it counts as 'missing' shelltest("zfs set autobackup:test=child test_source1/fs1") with OutputIO() as buf: with redirect_stdout(buf), redirect_stderr(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split( + " ")).run()) print(buf.getvalue()) - #should have done the snapshot cleanup for destoy missing: + # should have done the snapshot cleanup for destoy missing: self.assertIn("fs1@test-19101111000000: Destroying", buf.getvalue()) self.assertIn("fs1: Destroy missing: Still has children here.", buf.getvalue()) shelltest("zfs inherit autobackup:test test_source1/fs1") - with self.subTest("Normal destroyed leaf"): shelltest("zfs destroy -r test_source1/fs1/sub") - #wait for deadline of last snapshot + # wait for deadline of last snapshot with OutputIO() as buf: with redirect_stdout(buf): - #100y: lastest should not be old enough, while second to latest snapshot IS old enough: - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 100y".split(" ")).run()) + # 100y: lastest should not be old enough, while second to latest snapshot IS old enough: + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 100y".split( + " ")).run()) print(buf.getvalue()) self.assertIn(": Waiting for deadline", buf.getvalue()) - #past deadline, destroy + # past deadline, destroy with OutputIO() as buf: with redirect_stdout(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 1y".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 1y".split( + " ")).run()) print(buf.getvalue()) self.assertIn("sub: Destroying", buf.getvalue()) - with self.subTest("Leaf with other snapshot still using it"): shelltest("zfs destroy -r test_source1/fs1") shelltest("zfs snapshot -r test_target1/test_source1/fs1@other1") - with OutputIO() as buf: with redirect_stdout(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split( + " ")).run()) print(buf.getvalue()) - #cant finish because still in use: + # cant finish because still in use: self.assertIn("fs1: Destroy missing: Still in use", buf.getvalue()) shelltest("zfs destroy test_target1/test_source1/fs1@other1") - with self.subTest("In use by clone"): shelltest("zfs clone test_target1/test_source1/fs1@test-20101111000000 test_target1/clone1") with OutputIO() as buf: with redirect_stdout(buf), redirect_stderr(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split( + " ")).run()) print(buf.getvalue()) - #now tries to destroy our own last snapshot (before the final destroy of the dataset) + # now tries to destroy our own last snapshot (before the final destroy of the dataset) self.assertIn("fs1@test-20101111000000: Destroying", buf.getvalue()) - #but cant finish because still in use: + # but cant finish because still in use: self.assertIn("fs1: Error during --destroy-missing", buf.getvalue()) shelltest("zfs destroy test_target1/clone1") - with self.subTest("Should leave test_source1 parent"): - with OutputIO() as buf: with redirect_stdout(buf), redirect_stderr(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split( + " ")).run()) print(buf.getvalue()) - #should have done the snapshot cleanup for destoy missing: + # should have done the snapshot cleanup for destoy missing: self.assertIn("fs1: Destroying", buf.getvalue()) with OutputIO() as buf: with redirect_stdout(buf), redirect_stderr(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split( + " ")).run()) print(buf.getvalue()) - #on second run it sees the dangling ex-parent but doesnt know what to do with it (since it has no own snapshot) + # on second run it sees the dangling ex-parent but doesnt know what to do with it (since it has no own snapshot) self.assertIn("test_source1: Destroy missing: has no snapshots made by us", buf.getvalue()) - - - - #end result - r=shelltest("zfs list -H -o name -r -t all test_target1") - self.assertMultiLineEqual(r,""" + # end result + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") + self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 test_target1/test_source2 diff --git a/tests/test_encryption.py b/tests/test_encryption.py index 3193f26..7b705bf 100644 --- a/tests/test_encryption.py +++ b/tests/test_encryption.py @@ -301,7 +301,7 @@ test_target1/test_source2/fs2/sub encryptionroot - # NOTE: On some versions this leaves 2 weird sub-datasets that should'nt be there (its probably a zfs bug?) # so we ignore this, and just make sure the backup resumes correctly after reloading the key. - # r = shelltest("zfs get -r -t all encryptionroot test_target1") + # r = shelltest("zfs get -r -t snapshot,filesystem encryptionroot test_target1") # self.assertEqual(r, """ # NAME PROPERTY VALUE SOURCE # test_target1 encryptionroot - - diff --git a/tests/test_externalfailures.py b/tests/test_externalfailures.py index c14bb97..a76292d 100644 --- a/tests/test_externalfailures.py +++ b/tests/test_externalfailures.py @@ -51,7 +51,7 @@ class TestExternalFailures(unittest2.TestCase): self.assertIn(": resuming", buf.getvalue()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 @@ -94,7 +94,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # did we really resume? self.assertIn(": resuming", buf.getvalue()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 @@ -129,7 +129,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 @@ -164,7 +164,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 @@ -200,7 +200,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 self.assertIn("Aborting resume", buf.getvalue()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 @@ -259,7 +259,7 @@ test_target1/test_source2/fs2/sub@test-20101111000002 # with mocktime("20101111000000"): # self.assertFalse(ZfsAutobackup("test test_target1 --verbose --allow-empty --no-progress".split(" ")).run()) # -# r = shelltest("zfs list -H -o name -r -t all test_target1") +# r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") # # self.assertMultiLineEqual(r, """ # test_target1 diff --git a/tests/test_regressions.py b/tests/test_regressions.py index 273636f..87213f7 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -25,7 +25,7 @@ class TestZfsNode(unittest2.TestCase): "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty".split( " ")).run()) - r = shelltest("zfs list -H -o name -r -t bookmark,filesystem " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 diff --git a/tests/test_sendrecvpipes.py b/tests/test_sendrecvpipes.py index c552540..5c7b4f5 100644 --- a/tests/test_sendrecvpipes.py +++ b/tests/test_sendrecvpipes.py @@ -16,7 +16,8 @@ class TestSendRecvPipes(unittest2.TestCase): with self.subTest("local local pipe"): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", "--clear-mountpoint", + ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + "--clear-mountpoint", "--send-pipe=dd bs=1M", "--recv-pipe=dd bs=2M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -44,7 +45,7 @@ class TestSendRecvPipes(unittest2.TestCase): "--ssh-source=localhost", "--ssh-target=localhost", "--send-pipe=dd bs=1M", "--recv-pipe=dd bs=2M"]).run()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 @@ -85,7 +86,8 @@ test_target1/test_source2/fs2/sub@test-20101111000003 with self.subTest("local local pipe"): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", "--clear-mountpoint", "--buffer=1M"]).run()) + ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + "--clear-mountpoint", "--buffer=1M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -111,7 +113,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", "--ssh-source=localhost", "--ssh-target=localhost", "--buffer=1M"]).run()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 0a60477..c24b259 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -3,14 +3,14 @@ from zfs_autobackup.CmdPipe import CmdPipe from basetest import * import time -from zfs_autobackup.LogConsole import LogConsole +from zfs_autobackup.LogConsole import LogConsole class TestZfsAutobackup(unittest2.TestCase): def setUp(self): prepare_zpools() - self.longMessage=True + self.longMessage = True def test_invalidpars(self): @@ -18,7 +18,9 @@ class TestZfsAutobackup(unittest2.TestCase): with OutputIO() as buf: with redirect_stdout(buf): - self.assertEqual(ZfsAutobackup("test test_target1 --no-progress --resume --verbose --no-snapshot".split(" ")).run(), 0) + self.assertEqual( + ZfsAutobackup("test test_target1 --no-progress --resume --verbose --no-snapshot".split(" ")).run(), + 0) print(buf.getvalue()) self.assertIn("The --resume", buf.getvalue()) @@ -31,15 +33,14 @@ class TestZfsAutobackup(unittest2.TestCase): # correct message? self.assertIn("Please create this dataset", buf.getvalue()) - - def test_snapshotmode(self): + def test_snapshotmode(self): """test snapshot tool mode""" with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test --no-progress --verbose".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -54,27 +55,26 @@ test_source2/fs3/sub test_target1 """) - def test_defaults(self): - self.maxDiff=2000 + def test_defaults(self): + self.maxDiff = 2000 with self.subTest("no datasets selected"): with OutputIO() as buf: with redirect_stderr(buf): with mocktime("20101111000000"): - self.assertTrue(ZfsAutobackup("nonexisting test_target1 --verbose --debug --no-progress".split(" ")).run()) + self.assertTrue( + ZfsAutobackup("nonexisting test_target1 --verbose --debug --no-progress".split(" ")).run()) print(buf.getvalue()) - #correct message? + # correct message? self.assertIn("No source filesystems selected", buf.getvalue()) - with self.subTest("defaults with full verbose and debug"): - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --debug --no-progress".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -102,9 +102,8 @@ test_target1/test_source2/fs2/sub@test-20101111000000 with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --no-progress".split(" ")).run()) - - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -135,9 +134,8 @@ test_target1/test_source2/fs2/sub@test-20101111000001 """) with self.subTest("verify holds"): - - r=shelltest("zfs get -r userrefs test_source1 test_source2 test_target1") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs get -r userrefs test_source1 test_source2 test_target1") + self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_source1 userrefs - - test_source1/fs1 userrefs - - @@ -168,17 +166,20 @@ test_target1/test_source2/fs2/sub@test-20101111000000 userrefs 0 - test_target1/test_source2/fs2/sub@test-20101111000001 userrefs 1 - """) - #make sure time handling is correctly. try to make snapshots a year appart and verify that only snapshots mostly 1y old are kept - #So in this case we only want to see 2 snapshots of 2011, and none of the 2010's anymore. + # make sure time handling is correctly. try to make snapshots a year appart and verify that only snapshots mostly 1y old are kept + # So in this case we only want to see 2 snapshots of 2011, and none of the 2010's anymore. with self.subTest("test time checking"): with mocktime("20111211000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --verbose --no-progress".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1 --allow-empty --verbose --no-progress".split(" ")).run()) with mocktime("20111211000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --verbose --keep-source 1y1y --keep-target 1d1y --no-progress".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --allow-empty --verbose --keep-source 1y1y --keep-target 1d1y --no-progress".split( + " ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20111211000000 @@ -208,16 +209,16 @@ test_target1/test_source2/fs2/sub@test-20111211000000 test_target1/test_source2/fs2/sub@test-20111211000001 """) - def test_ignore_othersnaphots(self): + def test_ignore_othersnaphots(self): - r=shelltest("zfs snapshot test_source1/fs1@othersimple") - r=shelltest("zfs snapshot test_source1/fs1@otherdate-20001111000000") + r = shelltest("zfs snapshot test_source1/fs1@othersimple") + r = shelltest("zfs snapshot test_source1/fs1@otherdate-20001111000000") with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@othersimple @@ -243,16 +244,17 @@ test_target1/test_source2/fs2/sub test_target1/test_source2/fs2/sub@test-20101111000000 """) - def test_othersnaphots(self): + def test_othersnaphots(self): - r=shelltest("zfs snapshot test_source1/fs1@othersimple") - r=shelltest("zfs snapshot test_source1/fs1@otherdate-20001111000000") + r = shelltest("zfs snapshot test_source1/fs1@othersimple") + r = shelltest("zfs snapshot test_source1/fs1@otherdate-20001111000000") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --other-snapshots".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1 --no-progress --verbose --other-snapshots".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@othersimple @@ -280,16 +282,15 @@ test_target1/test_source2/fs2/sub test_target1/test_source2/fs2/sub@test-20101111000000 """) - - def test_nosnapshot(self): + def test_nosnapshot(self): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --no-progress".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - #(only parents are created ) - #TODO: it probably shouldn't create these - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + # (only parents are created ) + # TODO: it probably shouldn't create these + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1/sub @@ -304,14 +305,13 @@ test_target1/test_source2 test_target1/test_source2/fs2 """) - - def test_nosend(self): + def test_nosend(self): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-send --no-progress".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -326,15 +326,15 @@ test_source2/fs3/sub test_target1 """) - - def test_ignorereplicated(self): - r=shelltest("zfs snapshot test_source1/fs1@otherreplication") + def test_ignorereplicated(self): + r = shelltest("zfs snapshot test_source1/fs1@otherreplication") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --ignore-replicated".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1 --no-progress --verbose --ignore-replicated".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@otherreplication @@ -357,13 +357,13 @@ test_target1/test_source2/fs2/sub test_target1/test_source2/fs2/sub@test-20101111000000 """) - def test_noholds(self): + def test_noholds(self): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-holds --no-progress".split(" ")).run()) - r=shelltest("zfs get -r userrefs test_source1 test_source2 test_target1") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs get -r userrefs test_source1 test_source2 test_target1") + self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_source1 userrefs - - test_source1/fs1 userrefs - - @@ -388,14 +388,13 @@ test_target1/test_source2/fs2/sub userrefs - - test_target1/test_source2/fs2/sub@test-20101111000000 userrefs 0 - """) - - def test_strippath(self): + def test_strippath(self): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --strip-path=1 --no-progress".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -418,27 +417,28 @@ test_target1/fs2/sub@test-20101111000000 """) def test_strippath_collision(self): - with self.assertRaisesRegexp(Exception,"collision"): + with self.assertRaisesRegexp(Exception, "collision"): ZfsAutobackup("test test_target1 --verbose --strip-path=2 --no-progress --debug".split(" ")).run() def test_strippath_toomuch(self): - with self.assertRaisesRegexp(Exception,"too much"): + with self.assertRaisesRegexp(Exception, "too much"): ZfsAutobackup("test test_target1 --verbose --strip-path=3 --no-progress --debug".split(" ")).run() - def test_clearrefres(self): + def test_clearrefres(self): - #on zfs utils 0.6.x -x isnt supported - r=shelltest("zfs recv -x bla test >/dev/null /dev/null /dev/null /dev/null >> .*minutes left.*") + self.assertRegex(buf.getvalue(), ".*>>> .*minutes left.*") diff --git a/tests/test_zfsautobackup31.py b/tests/test_zfsautobackup31.py index e8650a7..f6dd7b0 100644 --- a/tests/test_zfsautobackup31.py +++ b/tests/test_zfsautobackup31.py @@ -1,23 +1,25 @@ from basetest import * import time + class TestZfsAutobackup31(unittest2.TestCase): """various new 3.1 features""" def setUp(self): prepare_zpools() - self.longMessage=True + self.longMessage = True def test_no_thinning(self): - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --keep-target=0 --keep-source=0 --no-thinning".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --keep-target=0 --keep-source=0 --no-thinning".split( + " ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -47,7 +49,6 @@ test_target1/test_source2/fs2/sub@test-20101111000000 test_target1/test_source2/fs2/sub@test-20101111000001 """) - def test_re_replication(self): """test re-replication of something thats already a backup (new in v3.1-beta5)""" @@ -55,16 +56,18 @@ test_target1/test_source2/fs2/sub@test-20101111000001 shelltest("zfs create test_target1/b") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1/a --no-progress --verbose --debug --allow-empty".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1/a --no-progress --verbose --debug --allow-empty".split(" ")).run()) - #NOTE: since v3.4 this changed. autobackup: properties are filtered. So its up to the admin to reset this property on the other side: + # NOTE: since v3.4 this changed. autobackup: properties are filtered. So its up to the admin to reset this property on the other side: shelltest("zfs set autobackup:test=true test_target1/a") with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1/b --no-progress --verbose --allow-empty".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1/b --no-progress --verbose --allow-empty".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t snapshot test_target1") - #NOTE: it wont backup test_target1/a/test_source2/fs2/sub to test_target1/b since it doesnt have the zfs_autobackup property anymore. + r = shelltest("zfs list -H -o name -r -t snapshot test_target1") + # NOTE: it wont backup test_target1/a/test_source2/fs2/sub to test_target1/b since it doesnt have the zfs_autobackup property anymore. self.assertMultiLineEqual(""" test_target1/a@test-20101111000001 test_target1/a/test_source1@test-20101111000001 @@ -92,10 +95,9 @@ test_target1/b/test_target1/a/test_source2@test-20101111000001 test_target1/b/test_target1/a/test_source2/fs2@test-20101111000001 test_target1/b/test_target1/a/test_source2/fs2/sub@test-20101111000000 test_target1/b/test_target1/a/test_source2/fs2/sub@test-20101111000001 -""",r) +""", r) def test_zfs_compressed(self): - with mocktime("20101111000000"): self.assertFalse( ZfsAutobackup("test test_target1 --no-progress --verbose --debug --zfs-compressed".split(" ")).run()) @@ -107,19 +109,18 @@ test_target1/b/test_target1/a/test_source2/fs2/sub@test-20101111000001 with mocktime("20101111000000"): self.assertFalse( - ZfsAutobackup("test test_target1 --no-progress --verbose --debug --force --strip-path=1".split(" ")).run()) + ZfsAutobackup( + "test test_target1 --no-progress --verbose --debug --force --strip-path=1".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t snapshot test_target1") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot test_target1") + self.assertMultiLineEqual(r, """ test_target1@test-20101111000000 test_target1/fs1@test-20101111000000 test_target1/fs1/sub@test-20101111000000 test_target1/fs2/sub@test-20101111000000 """) - def test_exclude_unchanged(self): - shelltest("zfs snapshot -r test_source1@somesnapshot") with mocktime("20101111000000"): @@ -127,7 +128,7 @@ test_target1/fs2/sub@test-20101111000000 ZfsAutobackup( "test test_target1 --verbose --allow-empty --exclude-unchanged=1".split(" ")).run()) - #everything should be excluded, but should not return an error (see #190) + # everything should be excluded, but should not return an error (see #190) with mocktime("20101111000001"): self.assertFalse( ZfsAutobackup( @@ -137,4 +138,3 @@ test_target1/fs2/sub@test-20101111000000 self.assertMultiLineEqual(r, """ test_target1/test_source2/fs2/sub@test-20101111000000 """) - diff --git a/tests/test_zfsautobackup32.py b/tests/test_zfsautobackup32.py index 068d322..cdb2bbe 100644 --- a/tests/test_zfsautobackup32.py +++ b/tests/test_zfsautobackup32.py @@ -1,29 +1,30 @@ from basetest import * + class TestZfsAutobackup32(unittest2.TestCase): """various new 3.2 features""" def setUp(self): prepare_zpools() - self.longMessage=True + self.longMessage = True def test_invalid_common_snapshot(self): - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - #create 2 snapshots with the same name, which are invalid as common snapshot + # create 2 snapshots with the same name, which are invalid as common snapshot shelltest("zfs snapshot test_source1/fs1@invalid") shelltest("zfs snapshot test_target1/test_source1/fs1@invalid") with mocktime("20101111000001"): - #try the old way (without guid checking), and fail: - self.assertEqual(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-guid-check".split(" ")).run(),1) - #new way should be ok: + # try the old way (without guid checking), and fail: + self.assertEqual(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --no-guid-check".split(" ")).run(), 1) + # new way should be ok: self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -56,23 +57,25 @@ test_target1/test_source2/fs2/sub@test-20101111000001 """) def test_invalid_common_snapshot_with_data(self): - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - #create 2 snapshots with the same name, which are invalid as common snapshot + # create 2 snapshots with the same name, which are invalid as common snapshot shelltest("zfs snapshot test_source1/fs1@invalid") shelltest("touch /test_target1/test_source1/fs1/shouldnotbeHere") shelltest("zfs snapshot test_target1/test_source1/fs1@invalid") with mocktime("20101111000001"): - #try the old way and fail: - self.assertEqual(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --destroy-incompatible --no-guid-check".split(" ")).run(),1) - #new way should be ok - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-incompatible".split(" ")).run()) + # try the old way and fail: + self.assertEqual(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --destroy-incompatible --no-guid-check".split( + " ")).run(), 1) + # new way should be ok + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-incompatible".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -103,8 +106,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 test_target1/test_source2/fs2/sub@test-20101111000001 """) - - #check consistent mounting behaviour, see issue #112 + # check consistent mounting behaviour, see issue #112 def test_mount_consitency_mounted(self): """only filesystems that have canmount=on with a mountpoint should be mounted. """ @@ -113,35 +115,32 @@ test_target1/test_source2/fs2/sub@test-20101111000001 with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - r=shelltest("zfs mount |grep -o /test_target1.*") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs mount |grep -o /test_target1.*") + self.assertMultiLineEqual(r, """ /test_target1 /test_target1/test_source1/fs1 /test_target1/test_source1/fs1/sub /test_target1/test_source2/fs2/sub """) - def test_mount_consitency_unmounted(self): """only test_target1 should be mounted in this test""" shelltest("zfs create -V 10M test_source1/fs1/subvol") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --clear-mountpoint".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --clear-mountpoint".split(" ")).run()) - r=shelltest("zfs mount |grep -o /test_target1.*") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs mount |grep -o /test_target1.*") + self.assertMultiLineEqual(r, """ /test_target1 """) - - - def test_transfer_thinning(self): # test pre/post/during transfer thinning and efficient transfer (no transerring of stuff that gets deleted on target) - #less output + # less output shelltest("zfs set autobackup:test2=true test_source1/fs1/sub") # nobody wants this one, will be destroyed before transferring (over a year ago) @@ -167,24 +166,24 @@ test_target1/test_source2/fs2/sub@test-20101111000001 with redirect_stdout(buf): # now do thinning and transfer all at once with mocktime("20010203000000"): - self.assertFalse(ZfsAutobackup("--keep-source=1d10d --keep-target=1m10m --allow-empty --verbose --clear-mountpoint --other-snapshots test2 test_target1".split(" ")).run()) - + self.assertFalse(ZfsAutobackup( + "--keep-source=1d10d --keep-target=1m10m --allow-empty --verbose --clear-mountpoint --other-snapshots test2 test_target1".split( + " ")).run()) print(buf.getvalue()) self.assertIn( -""" - [Source] test_source1/fs1/sub@test2-20000101000000: Destroying - [Source] test_source1/fs1/sub@test2-20010101000000: -> test_target1/test_source1/fs1/sub (new) - [Source] test_source1/fs1/sub@other1: -> test_target1/test_source1/fs1/sub - [Source] test_source1/fs1/sub@test2-20010101000000: Destroying - [Source] test_source1/fs1/sub@test2-20010201000000: -> test_target1/test_source1/fs1/sub - [Source] test_source1/fs1/sub@other2: -> test_target1/test_source1/fs1/sub - [Source] test_source1/fs1/sub@test2-20010203000000: -> test_target1/test_source1/fs1/sub -""", buf.getvalue()) + """ + [Source] test_source1/fs1/sub@test2-20000101000000: Destroying + [Source] test_source1/fs1/sub@test2-20010101000000: -> test_target1/test_source1/fs1/sub (new) + [Source] test_source1/fs1/sub@other1: -> test_target1/test_source1/fs1/sub + [Source] test_source1/fs1/sub@test2-20010101000000: Destroying + [Source] test_source1/fs1/sub@test2-20010201000000: -> test_target1/test_source1/fs1/sub + [Source] test_source1/fs1/sub@other2: -> test_target1/test_source1/fs1/sub + [Source] test_source1/fs1/sub@test2-20010203000000: -> test_target1/test_source1/fs1/sub + """, buf.getvalue()) - - r=shelltest("zfs list -H -o name -r -t snapshot test_source1 test_target1") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot test_source1 test_target1") + self.assertMultiLineEqual(r, """ test_source1/fs1/sub@other1 test_source1/fs1/sub@test2-20010201000000 test_source1/fs1/sub@other2 @@ -196,5 +195,3 @@ test_target1/test_source1/fs1/sub@test2-20010201000000 test_target1/test_source1/fs1/sub@other2 test_target1/test_source1/fs1/sub@test2-20010203000000 """) - - diff --git a/tests/test_zfsautobackup34.py b/tests/test_zfsautobackup34.py index ea7943b..68bc4a1 100644 --- a/tests/test_zfsautobackup34.py +++ b/tests/test_zfsautobackup34.py @@ -57,7 +57,7 @@ class TestZfsAutobackup34(unittest2.TestCase): with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 @@ -102,7 +102,7 @@ test_target1/test_source2/fs2/sub@test-20101111000002 with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - r = shelltest("zfs list -H -o name -r -t all test_source1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_source1") self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 @@ -118,7 +118,7 @@ test_source1/fs1/sub#test-20101111000001 self.assertFalse(ZfsAutobackup( "test test_target1 --no-progress --verbose --allow-empty --no-bookmarks".split(" ")).run()) - r = shelltest("zfs list -H -o name -r -t all test_source1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_source1") self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 @@ -133,7 +133,7 @@ test_source1/fs1/sub@test-20101111000002 with mocktime("20101111000003"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - r = shelltest("zfs list -H -o name -r -t all test_source1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_source1") self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 @@ -207,7 +207,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 # result: # for target a the bookmarks should be at 20101111000002, for target b the bookmarks should be at 20101111000003 - r = shelltest("zfs list -H -r -t all -o name " + TEST_POOLS) + r = shelltest("zfs list -H -r -t snapshot,filesystem -o name " + TEST_POOLS) self.assertRegexpMatches(r, """ test_source1 diff --git a/tests/test_zfscheck.py b/tests/test_zfscheck.py index cb239f3..38e5eb7 100644 --- a/tests/test_zfscheck.py +++ b/tests/test_zfscheck.py @@ -9,7 +9,6 @@ class TestZfsCheck(unittest2.TestCase): def setUp(self): pass - def test_volume(self): if exists("/.dockerenv"): @@ -23,23 +22,26 @@ class TestZfsCheck(unittest2.TestCase): with self.subTest("Generate"): with OutputIO() as buf: with redirect_stdout(buf): - self.assertFalse(ZfsCheck("test_source1/vol@test".split(" "),print_arguments=False).run()) + self.assertFalse(ZfsCheck("test_source1/vol@test".split(" "), print_arguments=False).run()) print(buf.getvalue()) self.assertEqual("""0 2c2ceccb5ec5574f791d45b63c940cff20550f9a 1 2c2ceccb5ec5574f791d45b63c940cff20550f9a """, buf.getvalue()) - #store on disk for next step, add one error. + # store on disk for next step, add one error. with open("/tmp/testhashes", "w") as fh: - fh.write(buf.getvalue()+"1\t2c2ceccb5ec5574f791d45b63c940cff20550f9X") + fh.write(buf.getvalue() + "1\t2c2ceccb5ec5574f791d45b63c940cff20550f9X") with self.subTest("Compare"): with OutputIO() as buf: with redirect_stdout(buf): - self.assertEqual(1, ZfsCheck("test_source1/vol@test --check=/tmp/testhashes".split(" "),print_arguments=False).run()) + self.assertEqual(1, ZfsCheck("test_source1/vol@test --check=/tmp/testhashes".split(" "), + print_arguments=False).run()) print(buf.getvalue()) - self.assertEqual("Chunk 1 failed: 2c2ceccb5ec5574f791d45b63c940cff20550f9X 2c2ceccb5ec5574f791d45b63c940cff20550f9a\n", buf.getvalue()) + self.assertEqual( + "Chunk 1 failed: 2c2ceccb5ec5574f791d45b63c940cff20550f9X 2c2ceccb5ec5574f791d45b63c940cff20550f9a\n", + buf.getvalue()) def test_filesystem(self): prepare_zpools() @@ -49,7 +51,7 @@ class TestZfsCheck(unittest2.TestCase): shelltest("mkdir /test_source1/dir") shelltest("cp tests/data/whole2 /test_source1/dir/testfile") - #it should ignore these: + # it should ignore these: shelltest("ln -s / /test_source1/symlink") shelltest("mknod /test_source1/c c 1 1") shelltest("mknod /test_source1/b b 1 1") @@ -67,17 +69,20 @@ class TestZfsCheck(unittest2.TestCase): dir/testfile 0 2e863f1fcccd6642e4e28453eba10d2d3f74d798 """, buf.getvalue()) - #store on disk for next step, add error + # store on disk for next step, add error with open("/tmp/testhashes", "w") as fh: - fh.write(buf.getvalue()+"dir/testfile 0 2e863f1fcccd6642e4e28453eba10d2d3f74d79X") + fh.write(buf.getvalue() + "dir/testfile 0 2e863f1fcccd6642e4e28453eba10d2d3f74d79X") with self.subTest("Compare"): with OutputIO() as buf: with redirect_stdout(buf): - self.assertEqual(1, ZfsCheck("test_source1@test --check=/tmp/testhashes".split(" "),print_arguments=False).run()) + self.assertEqual(1, ZfsCheck("test_source1@test --check=/tmp/testhashes".split(" "), + print_arguments=False).run()) print(buf.getvalue()) - self.assertEqual("dir/testfile: Chunk 0 failed: 2e863f1fcccd6642e4e28453eba10d2d3f74d79X 2e863f1fcccd6642e4e28453eba10d2d3f74d798\n", buf.getvalue()) + self.assertEqual( + "dir/testfile: Chunk 0 failed: 2e863f1fcccd6642e4e28453eba10d2d3f74d79X 2e863f1fcccd6642e4e28453eba10d2d3f74d798\n", + buf.getvalue()) def test_file(self): @@ -92,14 +97,17 @@ dir/testfile 0 2e863f1fcccd6642e4e28453eba10d2d3f74d798 # store on disk for next step, add error with open("/tmp/testhashes", "w") as fh: - fh.write(buf.getvalue()+"0 3c0bf91170d873b8e327d3bafb6bc074580d11bX") + fh.write(buf.getvalue() + "0 3c0bf91170d873b8e327d3bafb6bc074580d11bX") with self.subTest("Compare"): with OutputIO() as buf: with redirect_stdout(buf): - self.assertEqual(1,ZfsCheck("tests/data/whole --check=/tmp/testhashes".split(" "), print_arguments=False).run()) + self.assertEqual(1, ZfsCheck("tests/data/whole --check=/tmp/testhashes".split(" "), + print_arguments=False).run()) print(buf.getvalue()) - self.assertEqual("Chunk 0 failed: 3c0bf91170d873b8e327d3bafb6bc074580d11bX 3c0bf91170d873b8e327d3bafb6bc074580d11b7\n", buf.getvalue()) + self.assertEqual( + "Chunk 0 failed: 3c0bf91170d873b8e327d3bafb6bc074580d11bX 3c0bf91170d873b8e327d3bafb6bc074580d11b7\n", + buf.getvalue()) def test_tree(self): shelltest("rm -rf /tmp/testtree; mkdir /tmp/testtree") @@ -115,13 +123,12 @@ dir/testfile 0 2e863f1fcccd6642e4e28453eba10d2d3f74d798 with redirect_stdout(buf): self.assertFalse(ZfsCheck("/tmp/testtree --skip=1".split(" "), print_arguments=False).run()) - #since order varies, just check count (there is one empty line for some reason, only when testing like this) + # since order varies, just check count (there is one empty line for some reason, only when testing like this) print(buf.getvalue().split("\n")) - self.assertEqual(len(buf.getvalue().split("\n")),4) + self.assertEqual(len(buf.getvalue().split("\n")), 4) ###################################### with self.subTest("Compare, all incorrect, skip 1"): - # store on disk for next step, add error with open("/tmp/testhashes", "w") as fh: fh.write(""" @@ -134,13 +141,14 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bdX with OutputIO() as buf: with redirect_stdout(buf): - self.assertEqual(ZfsCheck("/tmp/testtree --check=/tmp/testhashes --skip=1".split(" "), print_arguments=False).run(), 3) + self.assertEqual(ZfsCheck("/tmp/testtree --check=/tmp/testhashes --skip=1".split(" "), + print_arguments=False).run(), 3) print(buf.getvalue()) self.assertMultiLineEqual("""partial: Chunk 0 failed: 642027d63bb0afd7e0ba197f2c66ad03e3d70deX 642027d63bb0afd7e0ba197f2c66ad03e3d70de1 whole2: Chunk 0 failed: 2e863f1fcccd6642e4e28453eba10d2d3f74d79X 2e863f1fcccd6642e4e28453eba10d2d3f74d798 whole_whole2_partial: Chunk 0 failed: 309ffffba2e1977d12f3b7469971f30d28b94bdX 309ffffba2e1977d12f3b7469971f30d28b94bd8 -""",buf.getvalue()) +""", buf.getvalue()) #################################### with self.subTest("Generate"): @@ -148,10 +156,10 @@ whole_whole2_partial: Chunk 0 failed: 309ffffba2e1977d12f3b7469971f30d28b94bdX 3 with redirect_stdout(buf): self.assertFalse(ZfsCheck("/tmp/testtree".split(" "), print_arguments=False).run()) - #file order on disk can vary, so sort it.. - sorted=buf.getvalue().split("\n") + # file order on disk can vary, so sort it.. + sorted = buf.getvalue().split("\n") sorted.sort() - sorted="\n".join(sorted)+"\n" + sorted = "\n".join(sorted) + "\n" print(sorted) self.assertEqual(""" @@ -184,11 +192,11 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bd8 shelltest("cp tests/data/whole /test_source1/testfile") shelltest("zfs snapshot test_source1@test") - #breaks pipe when head exists - #important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling + # breaks pipe when head exists + # important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling shelltest("python -m zfs_autobackup.ZfsCheck test_source1@test --debug | head -n1") - #should NOT be mounted anymore if cleanup went ok: + # should NOT be mounted anymore if cleanup went ok: self.assertNotRegex(shelltest("mount"), "test_source1@test") def test_brokenpipe_cleanup_volume(self): @@ -199,12 +207,12 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bd8 shelltest("zfs create -V200M test_source1/vol") shelltest("zfs snapshot test_source1/vol@test") - #breaks pipe when grep exists: - #important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling + # breaks pipe when grep exists: + # important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling shelltest("python -m zfs_autobackup.ZfsCheck test_source1/vol@test --debug| grep -m1 'Hashing file'") # time.sleep(1) - r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertMultiLineEqual(""" test_source1 test_source1/fs1 @@ -217,7 +225,4 @@ test_source2/fs2/sub test_source2/fs3 test_source2/fs3/sub test_target1 -""",r ) - - - +""", r) diff --git a/tests/test_zfsnode.py b/tests/test_zfsnode.py index 813abd4..dbfc03e 100644 --- a/tests/test_zfsnode.py +++ b/tests/test_zfsnode.py @@ -12,13 +12,15 @@ class TestZfsNode(unittest2.TestCase): def test_consistent_snapshot(self): logger = LogStub() description = "[Source]" - node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description) with self.subTest("first snapshot"): - (selected_datasets, excluded_datasets)=node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, - exclude_unchanged=0) + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], exclude_received=False, + exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-20101111000001", 100000) - r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertEqual(r, """ test_source1 test_source1/fs1 @@ -35,10 +37,11 @@ test_target1 """) with self.subTest("second snapshot, no changes, no snapshot"): - (selected_datasets, excluded_datasets)=node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, - exclude_unchanged=0) + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], exclude_received=False, + exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-20101111000002", 1) - r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertEqual(r, """ test_source1 test_source1/fs1 @@ -55,9 +58,11 @@ test_target1 """) with self.subTest("second snapshot, no changes, empty snapshot"): - (selected_datasets, excluded_datasets) =node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, exclude_unchanged=0) + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], exclude_received=False, + exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-20101111000002", 0) - r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertEqual(r, """ test_source1 test_source1/fs1 @@ -79,12 +84,16 @@ test_target1 def test_consistent_snapshot_prepostcmds(self): logger = LogStub() description = "[Source]" - node = ZfsNode(utc=False, snapshot_time_format="test", hold_name="test", logger=logger, description=description, debug_output=True) + node = ZfsNode(utc=False, snapshot_time_format="test", hold_name="test", logger=logger, description=description, + debug_output=True) with self.subTest("Test if all cmds are executed correctly (no failures)"): with OutputIO() as buf: with redirect_stdout(buf): - (selected_datasets, excluded_datasets) =node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, exclude_unchanged=0) + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], + exclude_received=False, + exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-1", 0, pre_snapshot_cmds=["echo pre1", "echo pre2"], @@ -96,12 +105,15 @@ test_target1 self.assertIn("STDOUT > post1", buf.getvalue()) self.assertIn("STDOUT > post2", buf.getvalue()) - - with self.subTest("Failure in the middle, only pre1 and both post1 and post2 should be executed, no snapshot should be attempted"): + with self.subTest( + "Failure in the middle, only pre1 and both post1 and post2 should be executed, no snapshot should be attempted"): with OutputIO() as buf: with redirect_stdout(buf): with self.assertRaises(ExecuteError): - (selected_datasets, excluded_datasets) =node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, exclude_unchanged=0) + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], + exclude_received=False, + exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-1", 0, pre_snapshot_cmds=["echo pre1", "false", "echo pre2"], @@ -118,8 +130,11 @@ test_target1 with OutputIO() as buf: with redirect_stdout(buf): with self.assertRaises(ExecuteError): - #same snapshot name as before so it fails - (selected_datasets, excluded_datasets) =node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, exclude_unchanged=0) + # same snapshot name as before so it fails + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], + exclude_received=False, + exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-1", 0, pre_snapshot_cmds=["echo pre1", "echo pre2"], @@ -136,20 +151,20 @@ test_target1 # Assert that timestamps keep relative order both for utc and for localtime logger = LogStub() description = "[Source]" - node_local = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) - node_utc = ZfsNode(utc=True, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) + node_local = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description) + node_utc = ZfsNode(utc=True, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description) for node in [node_local, node_utc]: with self.subTest("timestamp ordering " + ("utc" if node == node_utc else "localtime")): - dataset_a = ZfsDataset(node,"test_source1@test-20101111000001") - dataset_b = ZfsDataset(node,"test_source1@test-20101111000002") - dataset_c = ZfsDataset(node,"test_source1@test-20240101020202") + dataset_a = ZfsDataset(node, "test_source1@test-20101111000001") + dataset_b = ZfsDataset(node, "test_source1@test-20101111000002") + dataset_c = ZfsDataset(node, "test_source1@test-20240101020202") self.assertGreater(dataset_b.timestamp, dataset_a.timestamp) self.assertGreater(dataset_c.timestamp, dataset_b.timestamp) - def test_getselected(self): - # should be excluded by property shelltest("zfs create test_source1/fs1/subexcluded") shelltest("zfs set autobackup:test=false test_source1/fs1/subexcluded") @@ -165,9 +180,11 @@ test_target1 logger = LogStub() description = "[Source]" - node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) - (selected_datasets, excluded_datasets)=node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, - exclude_unchanged=1) + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description) + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], exclude_received=False, + exclude_unchanged=1) s = pformat(selected_datasets) print(s) @@ -177,11 +194,11 @@ test_target1 (local): test_source1/fs1/sub, (local): test_source2/fs2/sub]""") - def test_validcommand(self): logger = LogStub() description = "[Source]" - node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description) with self.subTest("test invalid option"): self.assertFalse(node.valid_command(["zfs", "send", "--invalid-option", "nonexisting"])) @@ -191,7 +208,8 @@ test_target1 def test_supportedsendoptions(self): logger = LogStub() description = "[Source]" - node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description) # -D propably always supported self.assertGreater(len(node.supported_send_options), 0) @@ -199,7 +217,8 @@ test_target1 logger = LogStub() description = "[Source]" # NOTE: this could hang via ssh if we dont close filehandles properly. (which was a previous bug) - node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description, ssh_to='localhost') + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description, ssh_to='localhost') self.assertIsInstance(node.supported_recv_options, list) From 4fbcc875587f2fd05452456eed7ae16d1fab185b Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 8 Oct 2024 18:00:15 +0200 Subject: [PATCH 042/135] fixes --- zfs_autobackup/ZfsDataset.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index fc6e3c7..51ca223 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -597,6 +597,7 @@ class ZfsDataset: ZfsDataset) Returns None if it cant find it. We try to find the bookmark with the preferred tag (which is usually a target path guid, to prevent conflicting bookmarks by multiple sends) + After that we return any bookmark that matches (and ignore the tag) Args: :rtype: ZfsDataset|None @@ -622,6 +623,17 @@ class ZfsDataset: return None + def find_exact_bookmark(self, bookmark_name): + """find exact bookmark name, or retruns none + + :rtype: ZfsDataset|None + """ + + for snapshot_bookmark in self.bookmarks: + if snapshot_bookmark.name == bookmark_name: + return snapshot_bookmark + return None + def find_snapshot_index(self, snapshot): """find snapshot index by snapshot (can be a snapshot_name or ZfsDataset) @@ -679,15 +691,21 @@ class ZfsDataset: return True def bookmark(self, tag): - """Bookmark this snapshot, and return the bookmark.""" + """Bookmark this snapshot, and return the bookmark. If the bookmark already exist, it returns it.""" # NOTE: we use the tag to add the target_path GUID, so that we can have multiple bookmarks for the same snapshot, but for different target. This is to make sure you can send a backup to two locations, without them interfering with eachothers bookmarks. if not self.is_snapshot: raise (Exception("Can only bookmark a snapshot!")) - self.debug("Bookmarking") - bookmark_name = self.prefix + "#" + self.tagless_suffix + self.zfs_node.tag_seperator + tag + + # does this exact bookmark (including tag) already exists? + existing_bookmark = self.find_exact_bookmark(bookmark_name) + if existing_bookmark is not None: + return existing_bookmark + + self.debug("Bookmarking {}".format(bookmark_name)) + cmd = [ "zfs", "bookmark", self.name, bookmark_name ] From 3919fe04a9cad2e5c099eb6f0051448702b2e936 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Fri, 11 Oct 2024 16:09:59 +0200 Subject: [PATCH 043/135] fixes --- tests/test_externalfailures.py | 2 +- zfs_autobackup/ZfsDataset.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_externalfailures.py b/tests/test_externalfailures.py index a76292d..58d26c9 100644 --- a/tests/test_externalfailures.py +++ b/tests/test_externalfailures.py @@ -158,7 +158,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # --test try again, should abort old resume with mocktime("20101111000002"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test --debug".split(" ")).run()) # try again, should abort old resume with mocktime("20101111000002"): diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 51ca223..bf95f65 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -692,7 +692,8 @@ class ZfsDataset: def bookmark(self, tag): """Bookmark this snapshot, and return the bookmark. If the bookmark already exist, it returns it.""" - # NOTE: we use the tag to add the target_path GUID, so that we can have multiple bookmarks for the same snapshot, but for different target. This is to make sure you can send a backup to two locations, without them interfering with eachothers bookmarks. + # NOTE: we use the tag to add the target_path GUID, so that we can have multiple bookmarks for the same snapshot, but for different targets. + # This is to make sure you can send a backup to two locations, without them interfering with eachothers bookmarks. if not self.is_snapshot: raise (Exception("Can only bookmark a snapshot!")) @@ -700,7 +701,7 @@ class ZfsDataset: bookmark_name = self.prefix + "#" + self.tagless_suffix + self.zfs_node.tag_seperator + tag # does this exact bookmark (including tag) already exists? - existing_bookmark = self.find_exact_bookmark(bookmark_name) + existing_bookmark = self.parent.find_exact_bookmark(bookmark_name) if existing_bookmark is not None: return existing_bookmark @@ -1423,7 +1424,9 @@ class ZfsDataset: # FIXME: met bookmarks kan de huidige snapshot na send ook meteen weg # FIXME: klopt niet, nu haalt ie altijd bookmark weg? wat als we naar andere target willen senden (zoals in test_encryption.py) if prev_source_snapshot_bookmark and ( + # its an obsolete snapshot? prev_source_snapshot_bookmark in source_obsoletes or ( + # its our bookmark? prev_source_snapshot_bookmark.is_bookmark and prev_source_snapshot_bookmark.tag == bookmark_tag)): prev_source_snapshot_bookmark.destroy() From ae2db2501d5858d2454fe01cdb1846439988a227 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Fri, 11 Oct 2024 16:14:09 +0200 Subject: [PATCH 044/135] fixes --- tests/test_zfsautobackup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index c24b259..b9bcd65 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -134,7 +134,7 @@ test_target1/test_source2/fs2/sub@test-20101111000001 """) with self.subTest("verify holds"): - r = shelltest("zfs get -r userrefs test_source1 test_source2 test_target1") + r = shelltest("zfs get -t filesystem,snapshot -r userrefs test_source1 test_source2 test_target1") self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_source1 userrefs - - @@ -362,7 +362,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-holds --no-progress".split(" ")).run()) - r = shelltest("zfs get -r userrefs test_source1 test_source2 test_target1") + r = shelltest("zfs get -t filesystem,snapshot -r userrefs test_source1 test_source2 test_target1") self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_source1 userrefs - - @@ -437,7 +437,7 @@ test_target1/fs2/sub@test-20101111000000 self.assertFalse( ZfsAutobackup("test test_target1 --no-progress --verbose --clear-refreservation".split(" ")).run()) - r = shelltest("zfs get -r refreservation test_source1 test_source2 test_target1") + r = shelltest("zfs get -t filesystem,snapshot -r refreservation test_source1 test_source2 test_target1") self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_source1 refreservation none default @@ -474,7 +474,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 refreservation - self.assertFalse( ZfsAutobackup("test test_target1 --no-progress --verbose --clear-mountpoint --debug".split(" ")).run()) - r = shelltest("zfs get -r canmount test_source1 test_source2 test_target1") + r = shelltest("zfs get -r canmount -t filesystem,snapshot test_source1 test_source2 test_target1") self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_source1 canmount on default From f3eb919b386687ccdf51d8f20f337614d7197755 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Mon, 14 Oct 2024 14:56:56 +0200 Subject: [PATCH 045/135] bump to v4.0 because of some major changes to how zfs-autobackup operates by default --- tests/test_regressions.py | 2 +- zfs_autobackup/CliBase.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_regressions.py b/tests/test_regressions.py index 87213f7..2c9bcb4 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -36,7 +36,7 @@ test_source2 test_source2/fs2 test_source2/fs2/sub test_source2/fs2/sub@test-20101111000002 -test_source2/fs3 +test_source2/fs31 test_source2/fs3/sub test_target1 test_target1/test_source1 diff --git a/zfs_autobackup/CliBase.py b/zfs_autobackup/CliBase.py index 0c069d2..6be5424 100644 --- a/zfs_autobackup/CliBase.py +++ b/zfs_autobackup/CliBase.py @@ -10,7 +10,7 @@ class CliBase(object): Overridden in subclasses that add stuff for the specific programs.""" # also used by setup.py - VERSION = "3.4-alpha" + VERSION = "4.0-alpha" HEADER = "{} v{} - (c)2022 E.H.Eefting (edwin@datux.nl)".format(os.path.basename(sys.argv[0]), VERSION) def __init__(self, argv, print_arguments=True): From 09370bc62a56f2fee3e1cdef222293719cb51064 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Mon, 14 Oct 2024 15:55:58 +0200 Subject: [PATCH 046/135] fixes --- tests/test_regressions.py | 56 +++++++++++++++++++++++++++++++++--- zfs_autobackup/ZfsDataset.py | 28 +++++++++--------- 2 files changed, 65 insertions(+), 19 deletions(-) diff --git a/tests/test_regressions.py b/tests/test_regressions.py index 2c9bcb4..ce87b08 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -9,6 +9,57 @@ class TestZfsNode(unittest2.TestCase): def test_keepsource0target10queuedsend(self): """Test if thinner doesnt destroy too much early on if there are no common snapshots YET. Issue #84""" + # old behaviour, without bookmarks + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send --no-bookmarks".split( + " ")).run()) + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send --no-bookmarks".split( + " ")).run()) + + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-bookmarks".split( + " ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000002 +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000002 +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000002 +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@test-20101111000000 +test_target1/test_source1/fs1@test-20101111000001 +test_target1/test_source1/fs1@test-20101111000002 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000000 +test_target1/test_source1/fs1/sub@test-20101111000001 +test_target1/test_source1/fs1/sub@test-20101111000002 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000000 +test_target1/test_source2/fs2/sub@test-20101111000001 +test_target1/test_source2/fs2/sub@test-20101111000002 +""") + + def test_keepsource0target10queuedsend_bookmarks(self): + """Test if thinner doesnt destroy too much early on if there are no common snapshots YET. Issue #84""" + # new behavior, with bookmarks. (will delete common snapshot, since there is a bookmark) with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( @@ -29,14 +80,11 @@ class TestZfsNode(unittest2.TestCase): self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 -test_source1/fs1@test-20101111000002 test_source1/fs1/sub -test_source1/fs1/sub@test-20101111000002 test_source2 test_source2/fs2 test_source2/fs2/sub -test_source2/fs2/sub@test-20101111000002 -test_source2/fs31 +test_source2/fs3 test_source2/fs3/sub test_target1 test_target1/test_source1 diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index bf95f65..8091ca2 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1377,7 +1377,6 @@ class ZfsDataset: write_embedded = False # now actually transfer the snapshots - do_rollback = rollback prev_source_snapshot_bookmark = source_common_snapshot prev_target_snapshot = target_dataset.find_snapshot(source_common_snapshot) @@ -1408,27 +1407,26 @@ class ZfsDataset: if prev_target_snapshot: prev_target_snapshot.release() - # bookmark common snapshot on source, or use holds if bookmarks are not enabled. if use_bookmarks: + # bookmark common snapshot, and clean up obsolete snapshots and bookmark source_bookmark = source_snapshot.bookmark(bookmark_tag) - # note: destroy source_snapshot when obsolete at this point? + if source_snapshot in source_obsoletes: + source_snapshot.destroy() + + if prev_source_snapshot_bookmark and prev_source_snapshot_bookmark.is_bookmark and prev_source_snapshot_bookmark.is_ours and prev_source_snapshot_bookmark.tag == bookmark_tag: + prev_source_snapshot_bookmark.destroy() + + # dont use bookmarks else: source_bookmark = None if holds: source_snapshot.hold() - if prev_source_snapshot_bookmark and prev_source_snapshot_bookmark.is_snapshot: - prev_source_snapshot_bookmark.release() - - # we may now destroy the previous source snapshot if its obsolete or an bookmark - # FIXME: met bookmarks kan de huidige snapshot na send ook meteen weg - # FIXME: klopt niet, nu haalt ie altijd bookmark weg? wat als we naar andere target willen senden (zoals in test_encryption.py) - if prev_source_snapshot_bookmark and ( - # its an obsolete snapshot? - prev_source_snapshot_bookmark in source_obsoletes or ( - # its our bookmark? - prev_source_snapshot_bookmark.is_bookmark and prev_source_snapshot_bookmark.tag == bookmark_tag)): - prev_source_snapshot_bookmark.destroy() + # release hold, cleanup obsolete snapshot + if prev_source_snapshot_bookmark and prev_source_snapshot_bookmark.is_snapshot: + prev_source_snapshot_bookmark.release() + if prev_source_snapshot_bookmark in source_obsoletes: + prev_source_snapshot_bookmark.destroy() # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, # the rest was already destroyed or will not be send) From b9d806a4c0d2ed05fc457a369e7803dfaf3a0ed2 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Mon, 14 Oct 2024 16:26:54 +0200 Subject: [PATCH 047/135] fixes --- tests/test_zfsautobackup.py | 1 + ...utobackup34.py => test_zfsautobackup40.py} | 57 +++++++++++-------- 2 files changed, 33 insertions(+), 25 deletions(-) rename tests/{test_zfsautobackup34.py => test_zfsautobackup40.py} (87%) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index b9bcd65..2b329b3 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -745,6 +745,7 @@ test_target1/test_source2/fs2/sub@test-20101111000001 def test_migrate(self): """test migration from other snapshotting systems. zfs-autobackup should be able to continue from any common snapshot, not just its own.""" + XXX migrate van bookmark ook testen shelltest("zfs snapshot test_source1/fs1@migrate1") shelltest("zfs create test_target1/test_source1") diff --git a/tests/test_zfsautobackup34.py b/tests/test_zfsautobackup40.py similarity index 87% rename from tests/test_zfsautobackup34.py rename to tests/test_zfsautobackup40.py index 68bc4a1..6f8c6c8 100644 --- a/tests/test_zfsautobackup34.py +++ b/tests/test_zfsautobackup40.py @@ -1,8 +1,8 @@ from basetest import * -class TestZfsAutobackup34(unittest2.TestCase): - """various new 3.4 features""" +class TestZfsAutobackup40(unittest2.TestCase): + """various new 4.0 features""" def setUp(self): prepare_zpools() @@ -44,36 +44,39 @@ class TestZfsAutobackup34(unittest2.TestCase): with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + target_guid = shelltest("zfs get -H -o value guid test_target1").strip() + # destroy stuff and see if it still selects the correct ones shelltest("zfs destroy test_source2/fs2/sub@test-20101111000001") - shelltest("zfs destroy test_source1/fs1/sub#test-20101111000001") + shelltest("zfs destroy test_source1/fs1/sub#test-20101111000001_" + target_guid) - # bookmark with incorrect GUID, should fallback to snapshot - shelltest("zfs destroy test_source1/fs1#test-20101111000001") + # replace bookmark with incorrect GUID, should fallback to snapshot + # NOTE: the incorrect bookmark should be ignored and not destroyed. + shelltest("zfs destroy test_source1/fs1#test-20101111000001_" + target_guid) shelltest("zfs snapshot test_source1/fs1@wrong") - shelltest("zfs bookmark test_source1/fs1@wrong \#test-20101111000001") + shelltest("zfs bookmark test_source1/fs1@wrong \#test-20101111000001_" + target_guid) shelltest("zfs destroy test_source1/fs1@wrong") with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) - self.assertMultiLineEqual(r, """ + r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + self.assertRegexpMatches(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000001 test_source1/fs1@test-20101111000002 -test_source1/fs1#test-20101111000001 -test_source1/fs1#test-20101111000002 +test_source1/fs1#test-20101111000001_[0-9]* +test_source1/fs1#test-20101111000002_[0-9]* test_source1/fs1/sub test_source1/fs1/sub@test-20101111000001 test_source1/fs1/sub@test-20101111000002 -test_source1/fs1/sub#test-20101111000002 +test_source1/fs1/sub#test-20101111000002_[0-9]* test_source2 test_source2/fs2 test_source2/fs2/sub test_source2/fs2/sub@test-20101111000002 -test_source2/fs2/sub#test-20101111000002 +test_source2/fs2/sub#test-20101111000002_[0-9]* test_source2/fs3 test_source2/fs3/sub test_target1 @@ -102,50 +105,54 @@ test_target1/test_source2/fs2/sub@test-20101111000002 with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_source1") - self.assertMultiLineEqual(r, """ + r = shelltest("zfs list -H -o name -r -t all test_source1") + self.assertRegexpMatches(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000001 -test_source1/fs1#test-20101111000001 +test_source1/fs1#test-20101111000001_[0-9]* test_source1/fs1/sub test_source1/fs1/sub@test-20101111000001 -test_source1/fs1/sub#test-20101111000001 +test_source1/fs1/sub#test-20101111000001_[0-9]* """) - # disable it + # disable it (this should not touch the old bookmark) with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( "test test_target1 --no-progress --verbose --allow-empty --no-bookmarks".split(" ")).run()) - r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_source1") - self.assertMultiLineEqual(r, """ + r = shelltest("zfs list -H -o name -r -t all test_source1") + self.assertRegexpMatches(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000001 test_source1/fs1@test-20101111000002 +test_source1/fs1#test-20101111000001_[0-9]* test_source1/fs1/sub test_source1/fs1/sub@test-20101111000001 test_source1/fs1/sub@test-20101111000002 +test_source1/fs1/sub#test-20101111000001_[0-9]* """) - # re-enable + # re-enable (now the old bookmark should be still left alone) with mocktime("20101111000003"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_source1") - self.assertMultiLineEqual(r, """ + r = shelltest("zfs list -H -o name -r -t all test_source1") + self.assertRegexpMatches(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000001 test_source1/fs1@test-20101111000002 test_source1/fs1@test-20101111000003 -test_source1/fs1#test-20101111000003 +test_source1/fs1#test-20101111000001_[0-9]* +test_source1/fs1#test-20101111000003_[0-9]* test_source1/fs1/sub test_source1/fs1/sub@test-20101111000001 test_source1/fs1/sub@test-20101111000002 test_source1/fs1/sub@test-20101111000003 -test_source1/fs1/sub#test-20101111000003 +test_source1/fs1/sub#test-20101111000001_[0-9]* +test_source1/fs1/sub#test-20101111000003_[0-9]* """) def test_tags(self): @@ -207,7 +214,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 # result: # for target a the bookmarks should be at 20101111000002, for target b the bookmarks should be at 20101111000003 - r = shelltest("zfs list -H -r -t snapshot,filesystem -o name " + TEST_POOLS) + r = shelltest("zfs list -H -r -t all -o name " + TEST_POOLS) self.assertRegexpMatches(r, """ test_source1 From 25b71ba032b263f45b6d497d9d929a30429ccf80 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Mon, 14 Oct 2024 17:01:39 +0200 Subject: [PATCH 048/135] now can migrate from any common snapshot or bookmark, even if the name mismatches --- tests/test_regressions.py | 47 ------------- tests/test_zfsautobackup.py | 1 - tests/test_zfsautobackup40.py | 127 ++++++++++++++++++++++++++++++++++ zfs_autobackup/ZfsDataset.py | 45 +++++++++--- 4 files changed, 161 insertions(+), 59 deletions(-) diff --git a/tests/test_regressions.py b/tests/test_regressions.py index ce87b08..2bcfa82 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -55,53 +55,6 @@ test_target1/test_source2/fs2/sub test_target1/test_source2/fs2/sub@test-20101111000000 test_target1/test_source2/fs2/sub@test-20101111000001 test_target1/test_source2/fs2/sub@test-20101111000002 -""") - - def test_keepsource0target10queuedsend_bookmarks(self): - """Test if thinner doesnt destroy too much early on if there are no common snapshots YET. Issue #84""" - # new behavior, with bookmarks. (will delete common snapshot, since there is a bookmark) - - with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send".split( - " ")).run()) - - with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send".split( - " ")).run()) - - with mocktime("20101111000002"): - self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty".split( - " ")).run()) - - r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) - self.assertMultiLineEqual(r, """ -test_source1 -test_source1/fs1 -test_source1/fs1/sub -test_source2 -test_source2/fs2 -test_source2/fs2/sub -test_source2/fs3 -test_source2/fs3/sub -test_target1 -test_target1/test_source1 -test_target1/test_source1/fs1 -test_target1/test_source1/fs1@test-20101111000000 -test_target1/test_source1/fs1@test-20101111000001 -test_target1/test_source1/fs1@test-20101111000002 -test_target1/test_source1/fs1/sub -test_target1/test_source1/fs1/sub@test-20101111000000 -test_target1/test_source1/fs1/sub@test-20101111000001 -test_target1/test_source1/fs1/sub@test-20101111000002 -test_target1/test_source2 -test_target1/test_source2/fs2 -test_target1/test_source2/fs2/sub -test_target1/test_source2/fs2/sub@test-20101111000000 -test_target1/test_source2/fs2/sub@test-20101111000001 -test_target1/test_source2/fs2/sub@test-20101111000002 """) def test_excludepaths(self): diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 2b329b3..b9bcd65 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -745,7 +745,6 @@ test_target1/test_source2/fs2/sub@test-20101111000001 def test_migrate(self): """test migration from other snapshotting systems. zfs-autobackup should be able to continue from any common snapshot, not just its own.""" - XXX migrate van bookmark ook testen shelltest("zfs snapshot test_source1/fs1@migrate1") shelltest("zfs create test_target1/test_source1") diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py index 6f8c6c8..a6ef3e9 100644 --- a/tests/test_zfsautobackup40.py +++ b/tests/test_zfsautobackup40.py @@ -283,3 +283,130 @@ test_target1/b/test_source2/fs2/sub@test-20101111000003 with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + def test_keepsource0target10queuedsend_bookmarks(self): + """Test if thinner doesnt destroy too much early on if there are no common snapshots YET. Issue #84""" + # new behavior, with bookmarks. (will delete common snapshot, since there is a bookmark) + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send".split( + " ")).run()) + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send".split( + " ")).run()) + + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty".split( + " ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1/sub +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@test-20101111000000 +test_target1/test_source1/fs1@test-20101111000001 +test_target1/test_source1/fs1@test-20101111000002 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000000 +test_target1/test_source1/fs1/sub@test-20101111000001 +test_target1/test_source1/fs1/sub@test-20101111000002 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000000 +test_target1/test_source2/fs2/sub@test-20101111000001 +test_target1/test_source2/fs2/sub@test-20101111000002 +""") + + def test_migrate_from_mismatching_bookmarkname(self): + """test migration from a bookmark with a mismatching name.""" + + shelltest("zfs snapshot test_source1/fs1@migrate1") + shelltest("zfs create test_target1/test_source1") + shelltest("zfs send test_source1/fs1@migrate1| zfs recv test_target1/test_source1/fs1") + + # make it so there is only the bookmark to migrate from, which a random non-matching name + shelltest("zfs bookmark test_source1/fs1@migrate1 \#randombookmarkname") + shelltest("zfs destroy test_source1/fs1@migrate1") + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --debug".split(" ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000000 +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000000 +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000000 +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@migrate1 +test_target1/test_source1/fs1@test-20101111000000 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000000 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000000 +""") + + def test_migrate_from_mismatching_snapshotname(self): + """test migration from a snapshot with a mismatching name.""" + + shelltest("zfs snapshot test_source1/fs1@migrate1") + shelltest("zfs create test_target1/test_source1") + shelltest("zfs send test_source1/fs1@migrate1| zfs recv test_target1/test_source1/fs1") + + # rename it so the names mismatch and guid matching is needed to resolve it + shelltest("zfs rename test_source1/fs1@migrate1 test_source1/fs1@randomsnapshotname") + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --debug".split(" ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@randomsnapshotname +test_source1/fs1@test-20101111000000 +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000000 +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000000 +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@migrate1 +test_target1/test_source1/fs1@test-20101111000000 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000000 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000000 +""") diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 8091ca2..22e7723 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -567,6 +567,24 @@ class ZfsDataset: return None + def find_guid_bookmark_snapshot(self, guid): + """find the first bookmark or snapshot that matches, prefers bookmarks. + Args: + :type guid:str + :rtype: ZfsDataset|None + """ + # Since this is slower, we only use it if the name matching with find_snapshot and find_bookmark doesn work. + + for bookmark in self.bookmarks: + if bookmark.properties['guid'] == guid: + return bookmark + + for snapshot in self.snapshots: + if snapshot.properties['guid'] == guid: + return snapshot + + return None + def find_snapshot(self, snapshot): """find snapshot by snapshot name (can be a suffix or a different ZfsDataset) Returns None if it cant find it. @@ -1066,30 +1084,27 @@ class ZfsDataset: On the source it prefers the specified bookmark_name - Args: - :rtype: ZfsDataset|None + :rtype: tuple[ZfsDataset, ZfsDataset] | tuple[None,None] :type guid_check: bool :type target_dataset: ZfsDataset :type bookmark_tag: str """ - bookmark = self.zfs_node.get_dataset(bookmark_tag) - if not target_dataset.exists or not target_dataset.snapshots: # target has nothing yet - return None + return None, None else: for target_snapshot in reversed(target_dataset.snapshots): - # Prefer bookmarks over snapshots + # Prefer bookmarks to snapshots source_bookmark = self.find_bookmark(target_snapshot, preferred_tag=bookmark_tag) if source_bookmark: if guid_check and source_bookmark.properties['guid'] != target_snapshot.properties['guid']: source_bookmark.warning("Bookmark has mismatching GUID, ignoring.") else: source_bookmark.debug("Common bookmark") - return source_bookmark + return source_bookmark, target_snapshot # Source snapshot with same suffix? source_snapshot = self.find_snapshot(target_snapshot) @@ -1098,7 +1113,12 @@ class ZfsDataset: source_snapshot.warning("Snapshot has mismatching GUID, ignoring.") else: source_snapshot.debug("Common snapshot") - return source_snapshot + return source_snapshot, target_snapshot + + # Extensive GUID search (slower but works with all names) + source_bookmark_snapshot = self.find_guid_bookmark_snapshot(target_snapshot.properties['guid']) + if source_bookmark_snapshot is not None: + return source_bookmark_snapshot, target_snapshot raise (Exception("Cant find common bookmark or snapshot with target.")) @@ -1221,9 +1241,12 @@ class ZfsDataset: ### 1: determine common and start snapshot target_dataset.debug("Determining start snapshot") - source_common_snapshot = self.find_common_snapshot(target_dataset, guid_check=guid_check, - bookmark_tag=bookmark_tag) - incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(source_common_snapshot, raw) + (source_common_snapshot, target_common_snapshot) = self.find_common_snapshot(target_dataset, + guid_check=guid_check, + bookmark_tag=bookmark_tag) + # if source_common_snapshot: + # source_common_snapshot.verbose("Common snapshot or bookmark") + incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(target_common_snapshot, raw) # let thinner decide whats obsolete on source after the transfer is done source_obsoletes = [] From 30c81b4f8a12d0fe3ab81ea23383e1c361aab796 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Mon, 14 Oct 2024 17:25:15 +0200 Subject: [PATCH 049/135] broke more stuff :) --- tests/test_zfsautobackup.py | 1 + tests/test_zfsautobackup40.py | 13 ++++++++++--- zfs_autobackup/ZfsDataset.py | 14 +++++++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index b9bcd65..2a77e5d 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -545,6 +545,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 canmount - - # should fail, now incompatible self.assertTrue(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + # FIXME: tries to get the guid of the non existing snapshot because of --test mode with mocktime("20101111000003"): # --test should succeed by destroying incompatibles self.assertFalse(ZfsAutobackup( diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py index a6ef3e9..1cd1349 100644 --- a/tests/test_zfsautobackup40.py +++ b/tests/test_zfsautobackup40.py @@ -343,7 +343,10 @@ test_target1/test_source2/fs2/sub@test-20101111000002 shelltest("zfs destroy test_source1/fs1@migrate1") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --debug".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test".split(" ")).run()) + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertMultiLineEqual(r, """ @@ -376,13 +379,17 @@ test_target1/test_source2/fs2/sub@test-20101111000000 shelltest("zfs snapshot test_source1/fs1@migrate1") shelltest("zfs create test_target1/test_source1") - shelltest("zfs send test_source1/fs1@migrate1| zfs recv test_target1/test_source1/fs1") + shelltest("zfs send test_source1/fs1@migrate1| zfs recv test_target1/test_source1/fs1") # rename it so the names mismatch and guid matching is needed to resolve it shelltest("zfs rename test_source1/fs1@migrate1 test_source1/fs1@randomsnapshotname") + # testmode with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --debug".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test".split(" ")).run()) + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertMultiLineEqual(r, """ diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 22e7723..882f943 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -653,7 +653,7 @@ class ZfsDataset: return None def find_snapshot_index(self, snapshot): - """find snapshot index by snapshot (can be a snapshot_name or + """find exact snapshot index by snapshot (can be a snapshot_name or ZfsDataset) Args: @@ -663,11 +663,11 @@ class ZfsDataset: if not isinstance(snapshot, ZfsDataset): snapshot_name = snapshot else: - snapshot_name = snapshot.tagless_suffix + snapshot_name = snapshot.suffix index = 0 for snapshot in self.snapshots: - if snapshot.tagless_suffix == snapshot_name: + if snapshot.suffix == snapshot_name: return index index = index + 1 @@ -1122,7 +1122,7 @@ class ZfsDataset: raise (Exception("Cant find common bookmark or snapshot with target.")) - def find_incompatible_snapshots(self, common_snapshot, raw): + def find_incompatible_snapshots(self, target_common_snapshot, raw): """returns a list[snapshots] that is incompatible for a zfs recv onto the common_snapshot. all direct followup snapshots with written=0 are compatible. @@ -1130,15 +1130,15 @@ class ZfsDataset: in raw-mode nothing is compatible. issue #219 Args: - :type common_snapshot: ZfsDataset + :type target_common_snapshot: ZfsDataset :type raw: bool """ ret = [] - if common_snapshot and self.snapshots: + if target_common_snapshot and self.snapshots: followup = True - for snapshot in self.snapshots[self.find_snapshot_index(common_snapshot) + 1:]: + for snapshot in self.snapshots[self.find_snapshot_index(target_common_snapshot) + 1:]: if raw or not followup or int(snapshot.properties['written']) != 0: followup = False ret.append(snapshot) From 1dd36b77d63cb31311c0f05d843d0eee1024d1cd Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 15 Oct 2024 11:56:56 +0200 Subject: [PATCH 050/135] fix --- tests/test_zfsautobackup.py | 3 +-- zfs_autobackup/ZfsDataset.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 2a77e5d..4bdc1bf 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -545,11 +545,10 @@ test_target1/test_source2/fs2/sub@test-20101111000000 canmount - - # should fail, now incompatible self.assertTrue(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - # FIXME: tries to get the guid of the non existing snapshot because of --test mode with mocktime("20101111000003"): # --test should succeed by destroying incompatibles self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --allow-empty --destroy-incompatible --test --debug".split( + "test test_target1 --no-progress --verbose --allow-empty --destroy-incompatible --test".split( " ")).run()) with mocktime("20101111000003"): diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 882f943..093c25e 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1116,9 +1116,14 @@ class ZfsDataset: return source_snapshot, target_snapshot # Extensive GUID search (slower but works with all names) - source_bookmark_snapshot = self.find_guid_bookmark_snapshot(target_snapshot.properties['guid']) - if source_bookmark_snapshot is not None: - return source_bookmark_snapshot, target_snapshot + try: + source_bookmark_snapshot = self.find_guid_bookmark_snapshot(target_snapshot.properties['guid']) + if source_bookmark_snapshot is not None: + return source_bookmark_snapshot, target_snapshot + except ExecuteError as e: + # in readonly mode we igore a failed property read for non existing snapshots + if not self.zfs_node.readonly: + raise e raise (Exception("Cant find common bookmark or snapshot with target.")) From 9dd46c9ea7ab64d26382da96ed65a21da3293ca9 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 15 Oct 2024 12:26:25 +0200 Subject: [PATCH 051/135] fix test --- tests/test_zfsautobackup.py | 23 ++++++++++++----------- tests/test_zfsautobackup40.py | 3 +++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 4bdc1bf..b7a0d8a 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -100,7 +100,8 @@ test_target1/test_source2/fs2/sub@test-20101111000000 with self.subTest("bare defaults, allow empty"): with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --no-progress".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1 --allow-empty --no-progress --no-bookmarks".split(" ")).run()) r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertMultiLineEqual(r, """ @@ -173,41 +174,41 @@ test_target1/test_source2/fs2/sub@test-20101111000001 userrefs 1 - self.assertFalse( ZfsAutobackup("test test_target1 --allow-empty --verbose --no-progress".split(" ")).run()) - with mocktime("20111211000001"): + with mocktime("20111212000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --allow-empty --verbose --keep-source 1y1y --keep-target 1d1y --no-progress".split( + "test test_target1 --allow-empty --verbose --keep-source 1d1y --keep-target 1d1y --no-progress".split( " ")).run()) r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) - self.assertMultiLineEqual(r, """ + self.assertMultiLineEqual(""" test_source1 test_source1/fs1 test_source1/fs1@test-20111211000000 -test_source1/fs1@test-20111211000001 +test_source1/fs1@test-20111212000000 test_source1/fs1/sub test_source1/fs1/sub@test-20111211000000 -test_source1/fs1/sub@test-20111211000001 +test_source1/fs1/sub@test-20111212000000 test_source2 test_source2/fs2 test_source2/fs2/sub test_source2/fs2/sub@test-20111211000000 -test_source2/fs2/sub@test-20111211000001 +test_source2/fs2/sub@test-20111212000000 test_source2/fs3 test_source2/fs3/sub test_target1 test_target1/test_source1 test_target1/test_source1/fs1 test_target1/test_source1/fs1@test-20111211000000 -test_target1/test_source1/fs1@test-20111211000001 +test_target1/test_source1/fs1@test-20111212000000 test_target1/test_source1/fs1/sub test_target1/test_source1/fs1/sub@test-20111211000000 -test_target1/test_source1/fs1/sub@test-20111211000001 +test_target1/test_source1/fs1/sub@test-20111212000000 test_target1/test_source2 test_target1/test_source2/fs2 test_target1/test_source2/fs2/sub test_target1/test_source2/fs2/sub@test-20111211000000 -test_target1/test_source2/fs2/sub@test-20111211000001 -""") +test_target1/test_source2/fs2/sub@test-20111212000000 +""", r) def test_ignore_othersnaphots(self): diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py index 1cd1349..468382e 100644 --- a/tests/test_zfsautobackup40.py +++ b/tests/test_zfsautobackup40.py @@ -417,3 +417,6 @@ test_target1/test_source2/fs2 test_target1/test_source2/fs2/sub test_target1/test_source2/fs2/sub@test-20101111000000 """) + +# TODO +# - check if holds and no-holds function ok when bookmarks are enabled From 832134ac3cbeef7c12a0666657f8bf0a61b0846f Mon Sep 17 00:00:00 2001 From: DatuX Date: Tue, 21 Jan 2025 11:11:05 +0100 Subject: [PATCH 052/135] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 022178e..f216ce3 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ # NOTE: this is the v3.4 development branch. Not ready for testing yet! +** This now will be the upcoming v4.0 release due to some major changes in how zfs-autobackup handles certain stuff ** + [![Tests](https://github.com/psy0rz/zfs_autobackup/workflows/Regression%20tests/badge.svg)](https://github.com/psy0rz/zfs_autobackup/actions?query=workflow%3A%22Regression+tests%22) [![Coverage Status](https://coveralls.io/repos/github/psy0rz/zfs_autobackup/badge.svg)](https://coveralls.io/github/psy0rz/zfs_autobackup) [![Python Package](https://github.com/psy0rz/zfs_autobackup/workflows/Upload%20Python%20Package/badge.svg)](https://pypi.org/project/zfs-autobackup/) [![CodeQL](https://github.com/psy0rz/zfs_autobackup/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/psy0rz/zfs_autobackup/actions/workflows/codeql-analysis.yml) From 561d3950a3d90262e1290eba1b4fc32566bf7ebd Mon Sep 17 00:00:00 2001 From: DatuX Date: Tue, 21 Jan 2025 11:11:22 +0100 Subject: [PATCH 053/135] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f216ce3..20632ff 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # NOTE: this is the v3.4 development branch. Not ready for testing yet! -** This now will be the upcoming v4.0 release due to some major changes in how zfs-autobackup handles certain stuff ** +**This now will be the upcoming v4.0 release due to some major changes in how zfs-autobackup handles certain stuff** [![Tests](https://github.com/psy0rz/zfs_autobackup/workflows/Regression%20tests/badge.svg)](https://github.com/psy0rz/zfs_autobackup/actions?query=workflow%3A%22Regression+tests%22) [![Coverage Status](https://coveralls.io/repos/github/psy0rz/zfs_autobackup/badge.svg)](https://coveralls.io/github/psy0rz/zfs_autobackup) [![Python Package](https://github.com/psy0rz/zfs_autobackup/workflows/Upload%20Python%20Package/badge.svg)](https://pypi.org/project/zfs-autobackup/) [![CodeQL](https://github.com/psy0rz/zfs_autobackup/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/psy0rz/zfs_autobackup/actions/workflows/codeql-analysis.yml) From 4bd3bd5d7563ae0500154b519acad05595ec48e6 Mon Sep 17 00:00:00 2001 From: DatuX Date: Wed, 5 Feb 2025 10:10:52 +0100 Subject: [PATCH 054/135] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 20632ff..ae85a5d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ZFS autobackup -# NOTE: this is the v3.4 development branch. Not ready for testing yet! +# NOTE: this is the v4.0 development branch. Not ready for testing yet! **This now will be the upcoming v4.0 release due to some major changes in how zfs-autobackup handles certain stuff** From c757feecec10e1f88830ace11e2bc1c775bbcdae Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Mon, 16 Jun 2025 23:07:25 +0200 Subject: [PATCH 055/135] fix keep0 tests with and without bookmarks --- tests/test_zfsautobackup.py | 16 +++---- tests/test_zfsautobackup40.py | 79 +++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index b7a0d8a..960ab2c 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -782,23 +782,25 @@ test_target1/test_source2/fs2/sub@test-20101111000000 """) def test_keep0(self): - """test if keep-source=0 and keep-target=0 dont delete common snapshot and break backup""" + """test if keep-source=0 and keep-target=0 dont delete common snapshot and break backup, LEGACY NO BOOKMARK MODE""" + # backup mode with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=0".split(" ")).run()) + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=0 --no-bookmark".split( + " ")).run()) - # make snapshot, shouldnt delete 0 + # snapshot mode, shouldnt delete 0 with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( "test --no-progress --verbose --keep-source=0 --keep-target=0 --allow-empty".split(" ")).run()) - # make snapshot 2, shouldnt delete 0 since it has holds, but will delete 1 since it has no holds + # snapshot mode, shouldnt delete 0 since it has holds, but will delete 1 since it has no holds with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( "test --no-progress --verbose --keep-source=0 --keep-target=0 --allow-empty".split(" ")).run()) - r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem,bookmark " + TEST_POOLS) self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 @@ -829,10 +831,10 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # make another backup but with no-holds. we should naturally endup with only number 3 with mocktime("20101111000003"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=0 --no-holds --allow-empty --debug".split( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=0 --no-holds --allow-empty --debug --no-bookmark".split( " ")).run()) - r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem,bookmark " + TEST_POOLS) self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py index 468382e..87ec739 100644 --- a/tests/test_zfsautobackup40.py +++ b/tests/test_zfsautobackup40.py @@ -416,6 +416,85 @@ test_target1/test_source2 test_target1/test_source2/fs2 test_target1/test_source2/fs2/sub test_target1/test_source2/fs2/sub@test-20101111000000 +""") + + def test_keep0_bookmarks(self): + """test if keep-source=0 and keep-target=0 only keeps a bookmark at the source. (contrary to legacy holds-mode)""" + + # backup mode + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=0".split( + " ")).run()) + + # snapshot mode, will delete 0 + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test --no-progress --verbose --keep-source=0 --keep-target=0 --allow-empty".split(" ")).run()) + + # snapshot mode, will detete 1 + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup( + "test --no-progress --verbose --keep-source=0 --keep-target=0 --allow-empty".split(" ")).run()) + + # now we have only bookmark 0 and snapshot 2 at source + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem,bookmark " + TEST_POOLS) + self.assertRegexpMatches(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000002 +test_source1/fs1#test-20101111000000_[0-9]* +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000002 +test_source1/fs1/sub#test-20101111000000_[0-9]* +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000002 +test_source2/fs2/sub#test-20101111000000_[0-9]* +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@test-20101111000000 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000000 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000000 +""") + + # make another backup now the source only has bookmarks of 3 left + with mocktime("20101111000003"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=0 --allow-empty --debug".split( + " ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem,bookmark " + TEST_POOLS) + self.assertRegexpMatches(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1#test-20101111000003_[0-9]* +test_source1/fs1/sub +test_source1/fs1/sub#test-20101111000003_[0-9]* +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub#test-20101111000003_[0-9]* +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@test-20101111000003 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000003 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000003 """) # TODO From ed9f3ecb61454b295f8733a4b02cc71142861e95 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Mon, 16 Jun 2025 23:09:56 +0200 Subject: [PATCH 056/135] github is retiring ubuntu 20 --- .github/workflows/regression.yml | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index b5be76b..50fb169 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -1,7 +1,7 @@ name: Regression tests -on: ["push", "pull_request"] +on: [ "push", "pull_request" ] @@ -26,23 +26,3 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: coveralls --service=github || true - ubuntu20: - runs-on: ubuntu-20.04 - - steps: - - name: Checkout - uses: actions/checkout@v3.5.0 - - - name: Prepare - run: sudo apt update && sudo apt install zfsutils-linux lzop pigz zstd gzip xz-utils lz4 mbuffer && sudo -H pip3 install coverage unittest2 mock==3.0.5 coveralls - - - - name: Regression test - run: sudo -E ./tests/run_tests - - - - name: Coveralls - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: coveralls --service=github || true - From 258203c3a60e24a8290a907f7b2156f646b51277 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 17 Jun 2025 00:10:28 +0200 Subject: [PATCH 057/135] bookmark tests. make sure we delete all bookmarks that look like we created --- tests/test_zfsautobackup.py | 4 ++- tests/test_zfsautobackup32.py | 22 ++++++------- tests/test_zfsautobackup40.py | 62 +++++++++++++++++++++++++++++++++++ zfs_autobackup/ZfsDataset.py | 6 +++- 4 files changed, 81 insertions(+), 13 deletions(-) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 960ab2c..7115d7c 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -895,7 +895,9 @@ test_target1/test_source2/fs2/sub@test-20101111000003 r = shelltest("zfs snapshot test_source1@test") l = LogConsole(show_verbose=True, show_debug=True, color=False) - n = ZfsNode(utc=False, snapshot_time_format="bla", hold_name="bla", logger=l) + n = ZfsNode(utc=False, snapshot_time_format="bla", hold_name="bla", logger=l, readonly=False, + debug_output=False, thinner=None, exclude_snapshot_patterns=None, ssh_to=None, ssh_config=None, + description=None, tag_seperator="_") d = ZfsDataset(n, "test_source1@test") sp = d.send_pipe([], prev_snapshot=None, resume_token=None, show_progress=True, raw=False, send_pipes=[], diff --git a/tests/test_zfsautobackup32.py b/tests/test_zfsautobackup32.py index cdb2bbe..08b68f3 100644 --- a/tests/test_zfsautobackup32.py +++ b/tests/test_zfsautobackup32.py @@ -138,7 +138,7 @@ test_target1/test_source2/fs2/sub@test-20101111000001 """) def test_transfer_thinning(self): - # test pre/post/during transfer thinning and efficient transfer (no transerring of stuff that gets deleted on target) + # test pre/post/during transfer thinning and efficient transfer (no transerring of stuff that gets deleted on target) NO BOOKMARKS LEGACY MODE # less output shelltest("zfs set autobackup:test2=true test_source1/fs1/sub") @@ -167,22 +167,22 @@ test_target1/test_source2/fs2/sub@test-20101111000001 # now do thinning and transfer all at once with mocktime("20010203000000"): self.assertFalse(ZfsAutobackup( - "--keep-source=1d10d --keep-target=1m10m --allow-empty --verbose --clear-mountpoint --other-snapshots test2 test_target1".split( + "--keep-source=1d10d --keep-target=1m10m --allow-empty --verbose --clear-mountpoint --other-snapshots --no-bookmarks test2 test_target1".split( " ")).run()) print(buf.getvalue()) self.assertIn( """ - [Source] test_source1/fs1/sub@test2-20000101000000: Destroying - [Source] test_source1/fs1/sub@test2-20010101000000: -> test_target1/test_source1/fs1/sub (new) - [Source] test_source1/fs1/sub@other1: -> test_target1/test_source1/fs1/sub - [Source] test_source1/fs1/sub@test2-20010101000000: Destroying - [Source] test_source1/fs1/sub@test2-20010201000000: -> test_target1/test_source1/fs1/sub - [Source] test_source1/fs1/sub@other2: -> test_target1/test_source1/fs1/sub - [Source] test_source1/fs1/sub@test2-20010203000000: -> test_target1/test_source1/fs1/sub - """, buf.getvalue()) + [Source] test_source1/fs1/sub@test2-20000101000000: Destroying + [Source] test_source1/fs1/sub@test2-20010101000000: -> test_target1/test_source1/fs1/sub (new) + [Source] test_source1/fs1/sub@other1: -> test_target1/test_source1/fs1/sub + [Source] test_source1/fs1/sub@test2-20010101000000: Destroying + [Source] test_source1/fs1/sub@test2-20010201000000: -> test_target1/test_source1/fs1/sub + [Source] test_source1/fs1/sub@other2: -> test_target1/test_source1/fs1/sub + [Source] test_source1/fs1/sub@test2-20010203000000: -> test_target1/test_source1/fs1/sub +""", buf.getvalue()) - r = shelltest("zfs list -H -o name -r -t snapshot test_source1 test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,bookmark test_source1 test_target1") self.assertMultiLineEqual(r, """ test_source1/fs1/sub@other1 test_source1/fs1/sub@test2-20010201000000 diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py index 87ec739..98fd763 100644 --- a/tests/test_zfsautobackup40.py +++ b/tests/test_zfsautobackup40.py @@ -495,7 +495,69 @@ test_target1/test_source2 test_target1/test_source2/fs2 test_target1/test_source2/fs2/sub test_target1/test_source2/fs2/sub@test-20101111000003 +""") + + def test_transfer_thinning_bookmarks(self): + # test pre/post/during transfer thinning and efficient transfer (no transerring of stuff that gets deleted on target) WITH BOOKMARK SUPPORT + + # less output + shelltest("zfs set autobackup:test2=true test_source1/fs1/sub") + + # nobody wants this one, will be destroyed before transferring (over a year ago) + with mocktime("20000101000000"): + self.assertFalse(ZfsAutobackup("test2 --allow-empty".split(" ")).run()) + + # only target wants this one (monthlys) + with mocktime("20010101000000"): + self.assertFalse(ZfsAutobackup("test2 --allow-empty".split(" ")).run()) + + # both want this one (dayly + monthly) + # other snapshots should influence the middle one that we actually want. + with mocktime("20010201000000"): + shelltest("zfs snapshot test_source1/fs1/sub@other1") + self.assertFalse(ZfsAutobackup("test2 --allow-empty".split(" ")).run()) + shelltest("zfs snapshot test_source1/fs1/sub@other2") + + # only source wants this one (dayly) + with mocktime("20010202000000"): + self.assertFalse(ZfsAutobackup("test2 --allow-empty".split(" ")).run()) + + with OutputIO() as buf: + with redirect_stdout(buf): + # now do thinning and transfer all at once + with mocktime("20010203000000"): + self.assertFalse(ZfsAutobackup( + "--keep-source=1d10d --keep-target=1m10m --allow-empty --verbose --clear-mountpoint --other-snapshots test2 test_target1".split( + " ")).run()) + + print(buf.getvalue()) + self.assertIn( + """ + [Source] test_source1/fs1/sub@test2-20000101000000: Destroying + [Source] test_source1/fs1/sub@test2-20010101000000: -> test_target1/test_source1/fs1/sub (new) + [Source] test_source1/fs1/sub@test2-20010101000000: Destroying + [Source] test_source1/fs1/sub@other1: -> test_target1/test_source1/fs1/sub + [Source] test_source1/fs1/sub@test2-20010201000000: -> test_target1/test_source1/fs1/sub + [Source] test_source1/fs1/sub@other2: -> test_target1/test_source1/fs1/sub + [Source] test_source1/fs1/sub@test2-20010203000000: -> test_target1/test_source1/fs1/sub +""", buf.getvalue()) + + # note: also important that it leaves only the last bookmark on the source. + r = shelltest("zfs list -H -o name -r -t snapshot,bookmark test_source1 test_target1") + self.assertRegexpMatches(r, """ +test_source1/fs1/sub@other1 +test_source1/fs1/sub@test2-20010201000000 +test_source1/fs1/sub@other2 +test_source1/fs1/sub@test2-20010202000000 +test_source1/fs1/sub@test2-20010203000000 +test_source1/fs1/sub#test2-20010203000000_[0-9]* +test_target1/test_source1/fs1/sub@test2-20010101000000 +test_target1/test_source1/fs1/sub@other1 +test_target1/test_source1/fs1/sub@test2-20010201000000 +test_target1/test_source1/fs1/sub@other2 +test_target1/test_source1/fs1/sub@test2-20010203000000 """) # TODO # - check if holds and no-holds function ok when bookmarks are enabled +# -delete bookmarks we have on other snapshots diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 093c25e..b07194b 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1441,9 +1441,13 @@ class ZfsDataset: if source_snapshot in source_obsoletes: source_snapshot.destroy() - if prev_source_snapshot_bookmark and prev_source_snapshot_bookmark.is_bookmark and prev_source_snapshot_bookmark.is_ours and prev_source_snapshot_bookmark.tag == bookmark_tag: + # TODO: make a better is_ours specially for bookmarks, as part of the next refactoring splitting in more classes + # delete any bookmark that ends in ours tag_seprator + tag. + if prev_source_snapshot_bookmark and prev_source_snapshot_bookmark.is_bookmark and prev_source_snapshot_bookmark.name.endswith( + self.zfs_node.tag_seperator + bookmark_tag): prev_source_snapshot_bookmark.destroy() + # dont use bookmarks else: source_bookmark = None From 926d08f60b027b45ad72f95f4c0567b6b9e3fbe7 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 17 Jun 2025 00:17:06 +0200 Subject: [PATCH 058/135] test fixes --- tests/test_zfscheck.py | 2 +- tests/test_zfsnode.py | 24 ++++++++++++++++-------- zfs_autobackup/ZfsCheck.py | 4 +++- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/test_zfscheck.py b/tests/test_zfscheck.py index 38e5eb7..d836208 100644 --- a/tests/test_zfscheck.py +++ b/tests/test_zfscheck.py @@ -212,7 +212,7 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bd8 shelltest("python -m zfs_autobackup.ZfsCheck test_source1/vol@test --debug| grep -m1 'Hashing file'") # time.sleep(1) - r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem,volume " + TEST_POOLS) self.assertMultiLineEqual(""" test_source1 test_source1/fs1 diff --git a/tests/test_zfsnode.py b/tests/test_zfsnode.py index dbfc03e..f1f2d70 100644 --- a/tests/test_zfsnode.py +++ b/tests/test_zfsnode.py @@ -13,7 +13,8 @@ class TestZfsNode(unittest2.TestCase): logger = LogStub() description = "[Source]" node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", - logger=logger, description=description) + logger=logger, description=description, debug_output=False, ssh_to=None, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') with self.subTest("first snapshot"): (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", @@ -85,7 +86,8 @@ test_target1 logger = LogStub() description = "[Source]" node = ZfsNode(utc=False, snapshot_time_format="test", hold_name="test", logger=logger, description=description, - debug_output=True) + debug_output=True, ssh_to=None, ssh_config=None, exclude_snapshot_patterns=None, thinner=None, + readonly=False, tag_seperator='_') with self.subTest("Test if all cmds are executed correctly (no failures)"): with OutputIO() as buf: @@ -152,9 +154,11 @@ test_target1 logger = LogStub() description = "[Source]" node_local = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", - logger=logger, description=description) + logger=logger, description=description, debug_output=False, ssh_to=None, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') node_utc = ZfsNode(utc=True, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", - logger=logger, description=description) + logger=logger, description=description, debug_output=False, ssh_to=None, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') for node in [node_local, node_utc]: with self.subTest("timestamp ordering " + ("utc" if node == node_utc else "localtime")): @@ -181,7 +185,8 @@ test_target1 logger = LogStub() description = "[Source]" node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", - logger=logger, description=description) + logger=logger, description=description, debug_output=False, ssh_to=None, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, exclude_unchanged=1) @@ -198,7 +203,8 @@ test_target1 logger = LogStub() description = "[Source]" node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", - logger=logger, description=description) + logger=logger, description=description, debug_output=False, ssh_to=None, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') with self.subTest("test invalid option"): self.assertFalse(node.valid_command(["zfs", "send", "--invalid-option", "nonexisting"])) @@ -209,7 +215,8 @@ test_target1 logger = LogStub() description = "[Source]" node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", - logger=logger, description=description) + logger=logger, description=description, debug_output=False, ssh_to=None, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') # -D propably always supported self.assertGreater(len(node.supported_send_options), 0) @@ -218,7 +225,8 @@ test_target1 description = "[Source]" # NOTE: this could hang via ssh if we dont close filehandles properly. (which was a previous bug) node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", - logger=logger, description=description, ssh_to='localhost') + logger=logger, description=description, ssh_to='localhost', debug_output=False, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') self.assertIsInstance(node.supported_recv_options, list) diff --git a/zfs_autobackup/ZfsCheck.py b/zfs_autobackup/ZfsCheck.py index 61d45fe..b2a4ea9 100644 --- a/zfs_autobackup/ZfsCheck.py +++ b/zfs_autobackup/ZfsCheck.py @@ -18,7 +18,9 @@ class ZfsCheck(CliBase): # NOTE: common options argument parsing are in CliBase super(ZfsCheck, self).__init__(argv, print_arguments) - self.node = ZfsNode(self.log, utc=self.args.utc, readonly=self.args.test, debug_output=self.args.debug_output) + self.node = ZfsNode(self.log, utc=self.args.utc, readonly=self.args.test, debug_output=self.args.debug_output, + ssh_to=None, ssh_config=None, exclude_snapshot_patterns=None, thinner=None, + tag_seperator='_', description="", hold_name=None, snapshot_time_format=None) self.block_hasher = BlockHasher(count=self.args.count, bs=self.args.block_size, skip=self.args.skip) From 69326b313d13c68aab4f2cf353747fddff4289f4 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 17 Jun 2025 00:34:46 +0200 Subject: [PATCH 059/135] test fixes --- tests/test_scaling.py | 92 +++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/tests/test_scaling.py b/tests/test_scaling.py index b93d9a8..b6abb21 100644 --- a/tests/test_scaling.py +++ b/tests/test_scaling.py @@ -2,14 +2,16 @@ from basetest import * from zfs_autobackup.ExecuteNode import ExecuteNode -run_orig=ExecuteNode.run -run_counter=0 +run_orig = ExecuteNode.run +run_counter = 0 + def run_count(*args, **kwargs): global run_counter - run_counter=run_counter+1 + run_counter = run_counter + 1 return (run_orig(*args, **kwargs)) + class TestZfsScaling(unittest2.TestCase): def setUp(self): @@ -19,83 +21,79 @@ class TestZfsScaling(unittest2.TestCase): def test_manysnapshots(self): """count the number of commands when there are many snapshots.""" - snapshot_count=100 + snapshot_count = 100 print("Creating many snapshots...") - s="" - for i in range(1970,1970+snapshot_count): - s=s+"zfs snapshot test_source1/fs1@test-{:04}1111000000;".format(i) + s = "" + for i in range(1970, 1970 + snapshot_count): + s = s + "zfs snapshot test_source1/fs1@test-{:04}1111000000;".format(i) shelltest(s) global run_counter - run_counter=0 - with patch.object(ExecuteNode,'run', run_count) as p: - + run_counter = 0 + with patch.object(ExecuteNode, 'run', run_count) as p: with mocktime("20101112000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --keep-source=10000 --keep-target=10000 --no-holds --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=10000 --keep-target=10000 --no-holds --allow-empty".split( + " ")).run()) - - #this triggers if you make a change with an impact of more than O(snapshot_count/2) - expected_runs=342 + # this triggers if you make a change with an impact of more than O(snapshot_count/2) + expected_runs = 549 print("EXPECTED RUNS: {}".format(expected_runs)) print("ACTUAL RUNS : {}".format(run_counter)) - self.assertLess(abs(run_counter-expected_runs), snapshot_count/2) - - - run_counter=0 - with patch.object(ExecuteNode,'run', run_count) as p: + self.assertLess(abs(run_counter - expected_runs), snapshot_count / 2) + run_counter = 0 + with patch.object(ExecuteNode, 'run', run_count) as p: with mocktime("20101112000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --keep-source=10000 --keep-target=10000 --no-holds --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=10000 --keep-target=10000 --no-holds --allow-empty".split( + " ")).run()) - - #this triggers if you make a change with a performance impact of more than O(snapshot_count/2) - expected_runs=47 + # this triggers if you make a change with a performance impact of more than O(snapshot_count/2) + expected_runs = 54 print("EXPECTED RUNS: {}".format(expected_runs)) print("ACTUAL RUNS : {}".format(run_counter)) - self.assertLess(abs(run_counter-expected_runs), snapshot_count/2) + self.assertLess(abs(run_counter - expected_runs), snapshot_count / 2) def test_manydatasets(self): """count the number of commands when when there are many datasets""" - dataset_count=100 + dataset_count = 100 print("Creating many datasets...") - s="" - for i in range(0,dataset_count): - s=s+"zfs create test_source1/fs1/{};".format(i) + s = "" + for i in range(0, dataset_count): + s = s + "zfs create test_source1/fs1/{};".format(i) shelltest(s) global run_counter - #first run - run_counter=0 - with patch.object(ExecuteNode,'run', run_count) as p: - + # first run + run_counter = 0 + with patch.object(ExecuteNode, 'run', run_count) as p: with mocktime("20101112000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) - - #this triggers if you make a change with an impact of more than O(snapshot_count/2)` - expected_runs=842 + # this triggers if you make a change with an impact of more than O(snapshot_count/2)` + expected_runs = 949 print("EXPECTED RUNS: {}".format(expected_runs)) print("ACTUAL RUNS: {}".format(run_counter)) - self.assertLess(abs(run_counter-expected_runs), dataset_count/2) - - - #second run, should have higher number of expected_runs - run_counter=0 - with patch.object(ExecuteNode,'run', run_count) as p: + self.assertLess(abs(run_counter - expected_runs), dataset_count / 2) + # second run, should have higher number of expected_runs + run_counter = 0 + with patch.object(ExecuteNode, 'run', run_count) as p: with mocktime("20101112000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) - - #this triggers if you make a change with a performance impact of more than O(snapshot_count/2) - expected_runs=1047 + # this triggers if you make a change with a performance impact of more than O(snapshot_count/2) + expected_runs = 1254 print("EXPECTED RUNS: {}".format(expected_runs)) print("ACTUAL RUNS: {}".format(run_counter)) - self.assertLess(abs(run_counter-expected_runs), dataset_count/2) + self.assertLess(abs(run_counter - expected_runs), dataset_count / 2) From 5357b1c6e19fd4eac6db6d0de89ba68dc1c3ac2e Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 17 Jun 2025 23:30:12 +0200 Subject: [PATCH 060/135] started zfs-class splitting refactoring (everything broken from now on until its done) --- setup.py | 3 +- zfs_autobackup/ZfsBookmark.py | 51 ++ zfs_autobackup/ZfsContainer.py | 864 +++++++++++++++++++++ zfs_autobackup/ZfsDataset.py | 1284 +------------------------------- zfs_autobackup/ZfsNode.py | 38 +- zfs_autobackup/ZfsSnapshot.py | 354 +++++++++ 6 files changed, 1312 insertions(+), 1282 deletions(-) create mode 100644 zfs_autobackup/ZfsBookmark.py create mode 100644 zfs_autobackup/ZfsContainer.py create mode 100644 zfs_autobackup/ZfsSnapshot.py diff --git a/setup.py b/setup.py index 26201ec..5ebd527 100644 --- a/setup.py +++ b/setup.py @@ -26,12 +26,11 @@ setuptools.setup( packages=setuptools.find_packages(), classifiers=[ - "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", ], - python_requires='>=2.7', + python_requires='>=3.0', install_requires=[ "colorama", "argparse" diff --git a/zfs_autobackup/ZfsBookmark.py b/zfs_autobackup/ZfsBookmark.py new file mode 100644 index 0000000..4878665 --- /dev/null +++ b/zfs_autobackup/ZfsBookmark.py @@ -0,0 +1,51 @@ +from .ZfsDataset import ZfsDataset +from .ExecuteNode import ExecuteError + + +class ZfsBookmark(ZfsDataset): + """A ZFS bookmark""" + + @property + def prefix(self): + (filesystem, snapshot) = self.name.split("#") + return filesystem + + @property + def suffix(self): + (filesystem, snapshot_name) = self.name.split("#") + return snapshot_name + + @property + def typed_suffix(self): + return "#" + self.suffix + + @property + def tagless_suffix(self): + """snapshot or bookmark part of the name, but without the tag. (if its our snapshot we remove the tag)""" + + if self.is_ours and self.zfs_node.tag_seperator in self.suffix: + return self.suffix.split(self.zfs_node.tag_seperator)[0] + else: + return self.suffix + + @property + def tag(self): + """the tag-part of a snapshot or bookmark, if any. can return None""" + + if self.is_ours and self.zfs_node.tag_seperator in self.suffix: + return self.suffix.split(self.zfs_node.tag_seperator)[1] + else: + return None + + @property + def parent(self): + """get parent dataset + + :rtype: ZfsContainer | None + """ + return self.zfs_node.get_dataset(self.prefix) + + def destroy(self, fail_exception=False): + + self.debug("Destroying") + return super().destroy(fail_exception=fail_exception) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py new file mode 100644 index 0000000..6a012dc --- /dev/null +++ b/zfs_autobackup/ZfsContainer.py @@ -0,0 +1,864 @@ +from .ZfsDataset import ZfsDataset +from .ExecuteNode import ExecuteError +from .ZfsSnapshot import ZfsSnapshot + + +class ZfsContainer(ZfsDataset): + """Either a ZFS Filesystem or ZFS Dataset""" + + @property + def snapshots(self): + """get all snapshots of this dataset + :rtype: list[ZfsSnapshot] + """ + ret = [] + + for snapshot in self.snapshots_bookmarks: + if snapshot is ZfsSnapshot: + ret.append(snapshot) + + return ret + + def find_incompatible_snapshots(self, target_common_snapshot, raw): + """returns a list[snapshots] that is incompatible for a zfs recv onto + the common_snapshot. all direct followup snapshots with written=0 are + compatible. + + in raw-mode nothing is compatible. issue #219 + + Args: + :type target_common_snapshot: ZfsDataset + :type raw: bool + """ + + ret = [] + + if target_common_snapshot and self.snapshots: + followup = True + for snapshot in self.snapshots[self.find_snapshot_index(target_common_snapshot) + 1:]: + if raw or not followup or int(snapshot.properties['written']) != 0: + followup = False + ret.append(snapshot) + + return ret + + @property + def written_since_ours(self): + """get number of bytes written since our last snapshot""" + + if self.__written_since_ours is None: + latest_snapshot = self.our_snapshots[-1] + + self.debug("Getting bytes written since our last snapshot") + cmd = ["zfs", "get", "-H", "-ovalue", "-p", "written@" + str(latest_snapshot), self.name] + + output = self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0]) + + self.__written_since_ours = int(output[0]) + + return self.__written_since_ours + + def is_changed_ours(self, min_changed_bytes=1): + """dataset is changed since OUR latest snapshot? + + Args: + :type min_changed_bytes: int + """ + + if min_changed_bytes == 0: + return True + + if not self.our_snapshots: + return True + + # NOTE: filesystems can have a very small amount written without actual changes in some cases + if self.written_since_ours < min_changed_bytes: + return False + + return True + + def find_snapshot(self, snapshot): + """find snapshot by snapshot name (can be a suffix or a different + ZfsDataset) Returns None if it cant find it. + + Note that matches with our own snapshots will be done tagless. + + Args: + :rtype: ZfsDataset|None + :type snapshot: str|ZfsDataset|None + """ + + if snapshot is None: + return None + + if not isinstance(snapshot, ZfsDataset): + tagless_suffix = snapshot + else: + tagless_suffix = snapshot.tagless_suffix + + for snapshot in self.snapshots: + if snapshot.tagless_suffix == tagless_suffix: + return snapshot + + return None + + def find_next_snapshot(self, snapshot_bookmark): + """find next snapshot in this dataset, according to snapshot or bookmark. None if it doesn't exist + Args: + :type snapshot_bookmark: ZfsDataset + """ + + if not self.is_dataset: + raise (Exception("Please call this on a dataset.")) + + found = False + for snapshot in self.snapshots_bookmarks: + if snapshot == snapshot_bookmark: + found = True + else: + if found and snapshot.is_snapshot: + return snapshot + + return None + + def mount(self, mount_point): + + self.debug("Mounting") + + cmd = [ + "mount", "-tzfs", self.name, mount_point + ] + + self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) + + def unmount(self, mount_point): + + self.debug("Unmounting") + + cmd = [ + "umount", mount_point + ] + + self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) + + @property + def bookmarks(self): + + ret = [] + + for bookmark in self.snapshots_bookmarks: + if bookmark.is_bookmark: + ret.append(bookmark) + + return ret + + def automount(self): + """Mount the dataset as if one did a zfs mount -a, but only for this dataset + Failure to mount doesnt result in an exception, but outputs errors to STDERR. + + """ + + self.debug("Auto mounting") + + if self.properties['type'] != "filesystem": + return + + if self.properties['canmount'] != 'on': + return + + if self.properties['mountpoint'] == 'legacy': + return + + if self.properties['mountpoint'] == 'none': + return + + if self.properties['encryption'] != 'off' and self.properties['keystatus'] == 'unavailable': + return + + self.zfs_node.run(["zfs", "mount", self.name], valid_exitcodes=[0, 1]) + + def abort_resume(self): + """abort current resume state""" + self.debug("Aborting resume") + self.zfs_node.run(["zfs", "recv", "-A", self.name]) + + def rollback(self): + """rollback to latest existing snapshot on this dataset""" + + for snapshot in reversed(self.snapshots): + if snapshot.exists: + snapshot.rollback() + return + + def thin_list(self, keeps=None, ignores=None): + """determines list[snapshots] that should be kept or deleted based on + the thinning schedule. cull the herd! + + returns: ( keeps, obsoletes ) + + Args: + :param keeps: list[snapshots] to always keep (usually the last) + :param ignores: snapshots to completely ignore (usually incompatible target snapshots that are going to be destroyed anyway) + :type keeps: list[ZfsDataset] + :type ignores: list[ZfsDataset] + """ + + if ignores is None: + ignores = [] + if keeps is None: + keeps = [] + + snapshots = [snapshot for snapshot in self.our_snapshots if snapshot not in ignores] + + return self.zfs_node.thin_list(snapshots, keep_snapshots=keeps) + + def thin(self, skip_holds=False): + """destroys snapshots according to thin_list, except last snapshot + + Args: + :type skip_holds: bool + """ + + (keeps, obsoletes) = self.thin_list(keeps=self.our_snapshots[-1:]) + for obsolete in obsoletes: + if skip_holds and obsolete.is_hold(): + obsolete.verbose("Keeping (common snapshot)") + else: + obsolete.destroy() + self.snapshots.remove(obsolete) + + @property + def recursive_datasets(self, types="filesystem,volume"): + """get all (non-snapshot) datasets recursively under us + + Args: + :type types: str + :rtype: list[ZfsDataset] + """ + + if self.__recursive_datasets is None: + self.debug("Getting all recursive datasets under us") + + names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ + "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name + ]) + + self.__recursive_datasets = self.zfs_node.get_datasets(names[1:], force_exists=True) + + return self.__recursive_datasets + + @property + def datasets(self, types="filesystem,volume"): + """get all (non-snapshot) datasets directly under us + + Args: + :type types: str + """ + + if self.__datasets is None: + self.debug("Getting all datasets under us") + + names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ + "zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name + ]) + + self.__datasets = self.zfs_node.get_datasets(names[1:], force_exists=True) + + return self.__datasets + + @property + def parent(self): + """get parent dataset + + :rtype: ZfsContainer | None + """ + stripped = self.rstrip_path(1) + if stripped: + return self.zfs_node.get_dataset(stripped) + else: + return None + + def create_filesystem(self, parents=False, unmountable=True): + """create a filesystem + + Args: + :type parents: bool + """ + + # recurse up + if parents and self.parent and not self.parent.exists: + self.parent.create_filesystem(parents, unmountable) + + cmd = ["zfs", "create"] + + if unmountable: + cmd.extend(["-o", "canmount=off"]) + + cmd.append(self.name) + self.zfs_node.run(cmd) + + self.force_exists = True + + @property + def snapshots_bookmarks(self): + """get all snapshots and bookmarks of this dataset (ordered by createtxg, so its suitable to determine incremental zfs send order) + :rtype: list[ZfsSnapshot|ZfsBookmark] + """ + + # cached? + if self.__snapshots_bookmarks is None: + self.debug("Getting snapshots and bookmarks") + + cmd = [ + "zfs", "list", "-d", "1", "-r", "-t", "snapshot,bookmark", "-H", "-o", "name", "-s", "createtxg", + self.name + ] + + self.__snapshots_bookmarks = self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), + force_exists=True) + + return self.__snapshots_bookmarks + + def cache_snapshot_bookmark(self, snapshot, force=False): + """Update our snapshot and bookmark cache (if we have any). Use force if you want to force the caching, potentially triggering a zfs list + Args: + :type snapshot: ZfsDataset + """ + + if force: + self.snapshots_bookmarks.append(snapshot) + + elif self.__snapshots_bookmarks is not None: + self.__snapshots_bookmarks.append(snapshot) + + @property + def our_snapshots(self): + """get list[snapshots] creates by us of this dataset""" + ret = [] + + for snapshot in self.snapshots: + if snapshot.is_ours: + ret.append(snapshot) + + return ret + + def find_snapshot_in_list(self, snapshots): + """return ZfsDataset from the list of snapshots, if it matches the snapshot_name. Otherwise None + Args: + :type snapshots: list[ZfsDataset] + :rtype: ZfsDataset|None + """ + + for snapshot in snapshots: + if snapshot.tagless_suffix == self.tagless_suffix: + return snapshot + + return None + + def find_guid_bookmark_snapshot(self, guid): + """find the first bookmark or snapshot that matches, prefers bookmarks. + Args: + :type guid:str + :rtype: ZfsDataset|None + """ + # Since this is slower, we only use it if the name matching with find_snapshot and find_bookmark doesn work. + + for bookmark in self.bookmarks: + if bookmark.properties['guid'] == guid: + return bookmark + + for snapshot in self.snapshots: + if snapshot.properties['guid'] == guid: + return snapshot + + return None + + def find_bookmark(self, snapshot_bookmark, preferred_tag): + """find bookmark by bookmark name (can be a suffix or a different + ZfsDataset) Returns None if it cant find it. + + We try to find the bookmark with the preferred tag (which is usually a target path guid, to prevent conflicting bookmarks by multiple sends) + After that we return any bookmark that matches (and ignore the tag) + + Args: + :rtype: ZfsDataset|None + :type snapshot_bookmark: str|ZfsDataset|None + :type ignore_tag: bool + """ + + if snapshot_bookmark is None: + return None + + if not isinstance(snapshot_bookmark, ZfsDataset): + tagless_suffix = snapshot_bookmark + else: + tagless_suffix = snapshot_bookmark.tagless_suffix + + for snapshot_bookmark in self.bookmarks: + if snapshot_bookmark.tagless_suffix == tagless_suffix and snapshot_bookmark.tag == preferred_tag: + return snapshot_bookmark + + for snapshot_bookmark in self.bookmarks: + if snapshot_bookmark.tagless_suffix == tagless_suffix: + return snapshot_bookmark + + return None + + def find_exact_bookmark(self, bookmark_name): + """find exact bookmark name, or retruns none + + :rtype: ZfsDataset|None + """ + + for snapshot_bookmark in self.bookmarks: + if snapshot_bookmark.name == bookmark_name: + return snapshot_bookmark + return None + + def find_snapshot_index(self, snapshot): + """find exact snapshot index by snapshot (can be a snapshot_name or + ZfsDataset) + + Args: + :type snapshot: str or ZfsDataset + """ + + if not isinstance(snapshot, ZfsDataset): + snapshot_name = snapshot + else: + snapshot_name = snapshot.suffix + + index = 0 + for snapshot in self.snapshots: + if snapshot.suffix == snapshot_name: + return index + index = index + 1 + + return None + + def find_common_snapshot(self, target_dataset, guid_check, bookmark_tag): + """find latest common snapshot/bookmark between us and target returns None if its + an initial transfer. + + On the source it prefers the specified bookmark_name + + Args: + :rtype: tuple[ZfsDataset, ZfsDataset] | tuple[None,None] + :type guid_check: bool + :type target_dataset: ZfsDataset + :type bookmark_tag: str + """ + + if not target_dataset.exists or not target_dataset.snapshots: + # target has nothing yet + return None, None + else: + for target_snapshot in reversed(target_dataset.snapshots): + + # Prefer bookmarks to snapshots + source_bookmark = self.find_bookmark(target_snapshot, preferred_tag=bookmark_tag) + if source_bookmark: + if guid_check and source_bookmark.properties['guid'] != target_snapshot.properties['guid']: + source_bookmark.warning("Bookmark has mismatching GUID, ignoring.") + else: + source_bookmark.debug("Common bookmark") + return source_bookmark, target_snapshot + + # Source snapshot with same suffix? + source_snapshot = self.find_snapshot(target_snapshot) + if source_snapshot: + if guid_check and source_snapshot.properties['guid'] != target_snapshot.properties['guid']: + source_snapshot.warning("Snapshot has mismatching GUID, ignoring.") + else: + source_snapshot.debug("Common snapshot") + return source_snapshot, target_snapshot + + # Extensive GUID search (slower but works with all names) + try: + source_bookmark_snapshot = self.find_guid_bookmark_snapshot(target_snapshot.properties['guid']) + if source_bookmark_snapshot is not None: + return source_bookmark_snapshot, target_snapshot + except ExecuteError as e: + # in readonly mode we igore a failed property read for non existing snapshots + if not self.zfs_node.readonly: + raise e + + raise (Exception("Cant find common bookmark or snapshot with target.")) + + def _pre_clean(self, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): + """cleanup old stuff before starting snapshot syncing + + Args: + :type source_common_snapshot: ZfsDataset + :type target_dataset: ZfsDataset + :type source_obsoletes: list[ZfsDataset] + :type target_obsoletes: list[ZfsDataset] + :type target_transfers: list[ZfsDataset] + """ + + # on source: delete all obsoletes that are not in target_transfers (except common snapshot, if its not a bookmark) + for source_snapshot in self.snapshots: + if (source_snapshot in source_obsoletes + and source_common_snapshot != source_snapshot + and source_snapshot.find_snapshot_in_list(target_transfers) is None): + source_snapshot.destroy() + + # on target: destroy everything thats obsolete, except common_snapshot + if target_dataset.exists: + for target_snapshot in target_dataset.snapshots: + if (target_snapshot in target_obsoletes) \ + and (not source_common_snapshot or ( + target_snapshot.tagless_suffix != source_common_snapshot.tagless_suffix)): + if target_snapshot.exists: + target_snapshot.destroy() + + def is_selected(self, value, source, inherited, exclude_received, exclude_paths, exclude_unchanged): + """determine if dataset should be selected for backup (called from + ZfsNode) + + Args: + :type exclude_paths: list[str] + :type value: str + :type source: str + :type inherited: bool + :type exclude_received: bool + :type exclude_unchanged: int + + :param value: Value of the zfs property ("false"/"true"/"child"/parent/"-") + :param source: Source of the zfs property ("local"/"received", "-") + :param inherited: True of the value/source was inherited from a higher dataset. + + Returns: True : Selected + False: Excluded + None: No property found + """ + + # sanity checks + if source not in ["local", "received", "-"]: + # probably a program error in zfs-autobackup or new feature in zfs + raise (Exception( + "{} autobackup-property has illegal source: '{}' (possible BUG)".format(self.name, source))) + + if value not in ["false", "true", "child", "parent", "-"]: + # user error + raise (Exception( + "{} autobackup-property has illegal value: '{}'".format(self.name, value))) + + # non specified, ignore + if value == "-": + return None + + # only select childs of this dataset, ignore + if value == "child" and not inherited: + return False + + # only select parent, no childs, ignore + if value == "parent" and inherited: + return False + + # manually excluded by property + if value == "false": + self.verbose("Excluded") + return False + + # from here on the dataset is selected by property, now do additional exclusion checks + + # our path starts with one of the excluded paths? + for exclude_path in exclude_paths: + # if self.name.startswith(exclude_path): + if (self.name + "/").startswith(exclude_path + "/"): + # too noisy for verbose + self.debug("Excluded (path in exclude list)") + return False + + if source == "received": + if exclude_received: + self.verbose("Excluded (dataset already received)") + return False + + if not self.is_changed(exclude_unchanged): + self.verbose("Excluded (by --exclude-unchanged)") + return False + + self.verbose("Selected") + return True + + def destroy(self, fail_exception=False): + + self.verbose("Destroying") + return super().destroy(fail_exception=fail_exception) + + def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, + ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, + no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, + use_bookmarks, bookmark_tag): + """sync this dataset's snapshots to target_dataset, while also thinning + out old snapshots along the way. + + Args: + :type send_pipes: list[str] + :type recv_pipes: list[str] + :type target_dataset: ZfsDataset + :type features: list[str] + :type show_progress: bool + :type filter_properties: list[str] + :type set_properties: list[str] + :type ignore_recv_exit_code: bool + :type holds: bool + :type rollback: bool + :type decrypt: bool + :type also_other_snapshots: bool + :type no_send: bool + :type guid_check: bool + :type use_bookmarks: bool + :type bookmark_tag: str + """ + + # self.verbose("-> {}".format(target_dataset)) + + # defaults for these settings if there is no encryption stuff going on: + send_properties = True + raw = False + write_embedded = True + + # source dataset encrypted? + if self.properties.get('encryption', 'off') != 'off': + # user wants to send it over decrypted? + if decrypt: + # when decrypting, zfs cant send properties + send_properties = False + else: + # keep data encrypted by sending it raw (including properties) + raw = True + + (source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, + incompatible_target_snapshots) = \ + self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, + guid_check=guid_check, raw=raw, bookmark_tag=bookmark_tag) + + # NOTE: we do a pre-clean because we dont want filesystems to fillup when backups keep failing. + # Also usefull with no_send to still cleanup stuff. + self._pre_clean( + source_common_snapshot=source_common_snapshot, target_dataset=target_dataset, + target_transfers=target_transfers, target_obsoletes=target_obsoletes, source_obsoletes=source_obsoletes) + + # handle incompatible stuff on target + target_dataset.handle_incompatible_snapshots(incompatible_target_snapshots, destroy_incompatible) + + # now actually transfer the snapshots, if we want + if no_send or len(target_transfers) == 0: + return + + # check if we can resume + resume_token = self._validate_resume_token(target_dataset, target_transfers[0]) + + (active_filter_properties, active_set_properties) = self.get_allowed_properties(filter_properties, + set_properties) + + # always filter properties that start with 'autobackup:' (https://github.com/psy0rz/zfs_autobackup/issues/221) + for property in self.properties: + if property.startswith('autobackup:'): + active_filter_properties.append(property) + + # encrypt at target? + if encrypt and not raw: + # filter out encryption properties to let encryption on the target take place + active_filter_properties.extend(["keylocation", "pbkdf2iters", "keyformat", "encryption"]) + write_embedded = False + + # now actually transfer the snapshots + do_rollback = rollback + prev_source_snapshot_bookmark = source_common_snapshot + prev_target_snapshot = target_dataset.find_snapshot(source_common_snapshot) + for target_snapshot in target_transfers: + + source_snapshot = self.find_snapshot(target_snapshot) + + # do the rollback, one time at first transfer + if do_rollback: + target_dataset.rollback() + do_rollback = False + + source_snapshot.transfer_snapshot(target_snapshot, features=features, + prev_snapshot=prev_source_snapshot_bookmark, show_progress=show_progress, + filter_properties=active_filter_properties, + set_properties=active_set_properties, + ignore_recv_exit_code=ignore_recv_exit_code, + resume_token=resume_token, write_embedded=write_embedded, raw=raw, + send_properties=send_properties, send_pipes=send_pipes, + recv_pipes=recv_pipes, zfs_compressed=zfs_compressed, force=force) + + resume_token = None + + # hold/release common snapshot on the target. + if holds: + target_snapshot.hold() + + if prev_target_snapshot: + prev_target_snapshot.release() + + if use_bookmarks: + # bookmark common snapshot, and clean up obsolete snapshots and bookmark + source_bookmark = source_snapshot.bookmark(bookmark_tag) + if source_snapshot in source_obsoletes: + source_snapshot.destroy() + + # TODO: make a better is_ours specially for bookmarks, as part of the next refactoring splitting in more classes + # delete any bookmark that ends in ours tag_seprator + tag. + if prev_source_snapshot_bookmark and prev_source_snapshot_bookmark.is_bookmark and prev_source_snapshot_bookmark.name.endswith( + self.zfs_node.tag_seperator + bookmark_tag): + prev_source_snapshot_bookmark.destroy() + + + # dont use bookmarks + else: + source_bookmark = None + if holds: + source_snapshot.hold() + + # release hold, cleanup obsolete snapshot + if prev_source_snapshot_bookmark and prev_source_snapshot_bookmark.is_snapshot: + prev_source_snapshot_bookmark.release() + if prev_source_snapshot_bookmark in source_obsoletes: + prev_source_snapshot_bookmark.destroy() + + # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, + # the rest was already destroyed or will not be send) + if prev_target_snapshot in target_obsoletes: + prev_target_snapshot.destroy() + + # we always try to use the bookmark during incremental send + if source_bookmark: + prev_source_snapshot_bookmark = source_bookmark + else: + prev_source_snapshot_bookmark = source_snapshot + + prev_target_snapshot = target_snapshot + + def handle_incompatible_snapshots(self, incompatible_target_snapshots, destroy_incompatible): + """destroy incompatbile snapshots on target before sync, or inform user + what to do + + Args: + :type incompatible_target_snapshots: list[ZfsDataset] + :type destroy_incompatible: bool + """ + + if incompatible_target_snapshots: + if not destroy_incompatible: + for snapshot in incompatible_target_snapshots: + snapshot.error("Incompatible snapshot") + raise (Exception("Please destroy incompatible snapshots on target, or use --destroy-incompatible.")) + else: + for snapshot in incompatible_target_snapshots: + snapshot.verbose("Incompatible snapshot") + snapshot.destroy(fail_exception=True) + ### XXX: hacky, maybe not do it and have things check snapshot.exists instead? + self.snapshots.remove(snapshot) + + if len(incompatible_target_snapshots) > 0: + self.rollback() + + def _plan_sync(self, target_dataset, also_other_snapshots, guid_check, raw, bookmark_tag): + """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. + + Args: + :rtype: ( ZfsDataset, list[ZfsDataset], list[ZfsDataset], list[ZfsDataset], list[ZfsDataset] ) + :type target_dataset: ZfsDataset + :type also_other_snapshots: bool + :type guid_check: bool + :type raw: bool + :type bookmark_tag: str + + Returns: + tuple: A tuple containing: + - ZfsDataset: The common snapshot + - list[ZfsDataset]: Our obsolete source snapshots, after transfer is done. (will be thinned asap) + - list[ZfsDataset]: Our obsolete target snapshots, after transfer is done. (will be thinned asap) + - list[ZfsDataset]: Transfer target snapshots. These need to be transferred. + - list[ZfsDataset]: Incompatible target snapshots. Target snapshots that are in the way, after the common snapshot. (need to be destroyed to continue) + + + """ + + ### 1: determine common and start snapshot + + target_dataset.debug("Determining start snapshot") + (source_common_snapshot, target_common_snapshot) = self.find_common_snapshot(target_dataset, + guid_check=guid_check, + bookmark_tag=bookmark_tag) + # if source_common_snapshot: + # source_common_snapshot.verbose("Common snapshot or bookmark") + incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(target_common_snapshot, raw) + + # let thinner decide whats obsolete on source after the transfer is done + source_obsoletes = [] + if self.our_snapshots: + source_obsoletes = self.thin_list()[1] + + ### 2: Determine possible target snapshots + + # start with snapshots that already exist, minus imcompatibles + if target_dataset.exists: + possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots if + snapshot not in incompatible_target_snapshots] + else: + possible_target_snapshots = [] + + # add all snapshots from the source, starting after the common snapshot if it exists + if source_common_snapshot: + source_snapshot = self.find_next_snapshot(source_common_snapshot) + else: + if self.snapshots: + source_snapshot = self.snapshots[0] + else: + source_snapshot = None + + while source_snapshot: + # we want it? + if (also_other_snapshots or source_snapshot.is_ours) and not source_snapshot.is_snapshot_excluded: + # create virtual target snapshot + target_snapshot = target_dataset.zfs_node.get_dataset( + target_dataset.prefix + source_snapshot.typed_suffix, force_exists=False) + possible_target_snapshots.append(target_snapshot) + source_snapshot = self.find_next_snapshot(source_snapshot) + + ### 3: Let the thinner decide what it wants by looking at all the possible target_snaphots at once. + # always keep the last target snapshot as common snapshot. + if possible_target_snapshots: + (target_keeps, target_obsoletes) = target_dataset.zfs_node.thin_list(possible_target_snapshots, + keep_snapshots=[ + possible_target_snapshots[-1]]) + else: + target_keeps = [] + target_obsoletes = [] + + ### 4: Look at what the thinner wants and create a list of snapshots we still need to transfer + target_transfers = [] + for target_keep in target_keeps: + if not target_keep.exists: + target_transfers.append(target_keep) + + return source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots + + def _validate_resume_token(self, target_dataset, start_snapshot): + """validate and get (or destory) resume token + + Args: + :type target_dataset: ZfsDataset + :type start_snapshot: ZfsDataset + """ + + if target_dataset.exists and 'receive_resume_token' in target_dataset.properties: + if start_snapshot is None: + target_dataset.verbose("Aborting resume, its obsolete.") + target_dataset.abort_resume() + else: + resume_token = target_dataset.properties['receive_resume_token'] + # not valid anymore? + resume_snapshot = self.get_resume_snapshot(resume_token) + if not resume_snapshot or start_snapshot.suffix != resume_snapshot.suffix: + target_dataset.verbose("Aborting resume, its no longer valid.") + target_dataset.abort_resume() + else: + return resume_token diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index b07194b..9f08bb5 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -6,16 +6,10 @@ import time from .ExecuteNode import ExecuteError -# NOTE: get/create instances via zfs_node.get_dataset(). This is to make sure there is only one ZfsDataset object per actual dataset. - class ZfsDataset: - """a zfs dataset (filesystem/volume/snapshot/clone) Note that a dataset - doesn't have to actually exist (yet/anymore) Also most properties are cached - for performance-reasons, but also to allow --test to function correctly. + """A generic ZFS dataset, this has all the common functions of zfs filesystems, volumes, snapshots and bookmarks. - A dataset 'name' is the full name as you would use it with zfs list. - Internally we use it in 3 parts: the prefix, suffix and tag. - (see respective properties below) + It is used as a base class for ZfsSnapshot, ZfsBookmark and ZfsContainer. This class should not be instantiated directly. """ # illegal properties per dataset type. these will be removed from --set-properties and --filter-properties @@ -125,201 +119,6 @@ class ZfsDataset: """ return "/".join(self.split_path()[:-count]) - @property - def prefix(self): - """filesystem part of the name (before the @)""" - if self.is_snapshot: - (filesystem, snapshot) = self.name.split("@") - return filesystem - elif self.is_bookmark: - (filesystem, snapshot) = self.name.split("#") - return filesystem - else: - return self.name - - @property - def suffix(self): - """snapshot or bookmark part of the name (the stuff after the # or @) """ - if self.is_snapshot: - (filesystem, snapshot_name) = self.name.split("@") - return snapshot_name - elif self.is_bookmark: - (filesystem, bookmark_name) = self.name.split("#") - return bookmark_name - - raise (Exception("This is not a snapshot or bookmark")) - - @property - def tagless_suffix(self): - """snapshot or bookmark part of the name, but without the tag. (if its our snapshot we remove the tag)""" - - suffix = self.suffix - if self.is_ours and self.zfs_node.tag_seperator in suffix: - return suffix.split(self.zfs_node.tag_seperator)[0] - else: - return suffix - - @property - def typed_suffix(self): - """suffix with @ or # in front of it""" - - if self.is_snapshot: - (filesystem, snapshot_name) = self.name.split("@") - return "@" + snapshot_name - elif self.is_bookmark: - (filesystem, bookmark_name) = self.name.split("#") - return "#" + bookmark_name - - raise (Exception("This is not a snapshot or bookmark")) - - @property - def tag(self): - """the tag-part of a snapshot or bookmark, if any. can return None""" - - if self.is_ours and self.zfs_node.tag_seperator in self.suffix: - return self.suffix.split(self.zfs_node.tag_seperator)[1] - else: - return None - - @property - def is_snapshot(self): - """true if this dataset is a snapshot""" - return self.name.find("@") != -1 - - @property - def is_bookmark(self): - """true if this dataset is a bookmark""" - return self.name.find("#") != -1 - - @property - def is_dataset(self): - - return not (self.is_snapshot or self.is_bookmark) - - @property - def is_snapshot_excluded(self): - """true if this dataset is a snapshot and matches the exclude pattern""" - if not self.is_snapshot: - return False - - for pattern in self.zfs_node.exclude_snapshot_patterns: - if pattern.search(self.suffix) is not None: - self.debug("Excluded (path matches snapshot exclude pattern)") - return True - - def is_selected(self, value, source, inherited, exclude_received, exclude_paths, exclude_unchanged): - """determine if dataset should be selected for backup (called from - ZfsNode) - - Args: - :type exclude_paths: list[str] - :type value: str - :type source: str - :type inherited: bool - :type exclude_received: bool - :type exclude_unchanged: int - - :param value: Value of the zfs property ("false"/"true"/"child"/parent/"-") - :param source: Source of the zfs property ("local"/"received", "-") - :param inherited: True of the value/source was inherited from a higher dataset. - - Returns: True : Selected - False: Excluded - None: No property found - """ - - # sanity checks - if source not in ["local", "received", "-"]: - # probably a program error in zfs-autobackup or new feature in zfs - raise (Exception( - "{} autobackup-property has illegal source: '{}' (possible BUG)".format(self.name, source))) - - if value not in ["false", "true", "child", "parent", "-"]: - # user error - raise (Exception( - "{} autobackup-property has illegal value: '{}'".format(self.name, value))) - - # non specified, ignore - if value == "-": - return None - - # only select childs of this dataset, ignore - if value == "child" and not inherited: - return False - - # only select parent, no childs, ignore - if value == "parent" and inherited: - return False - - # manually excluded by property - if value == "false": - self.verbose("Excluded") - return False - - # from here on the dataset is selected by property, now do additional exclusion checks - - # our path starts with one of the excluded paths? - for exclude_path in exclude_paths: - # if self.name.startswith(exclude_path): - if (self.name + "/").startswith(exclude_path + "/"): - # too noisy for verbose - self.debug("Excluded (path in exclude list)") - return False - - if source == "received": - if exclude_received: - self.verbose("Excluded (dataset already received)") - return False - - if not self.is_changed(exclude_unchanged): - self.verbose("Excluded (by --exclude-unchanged)") - return False - - self.verbose("Selected") - return True - - @property - def parent(self): - """get zfs-parent of this dataset. for snapshots this means it will get - the filesystem/volume that it belongs to. otherwise it will return the - parent according to path - - we cache this so everything in the parent that is cached also stays. - - returns None if there is no parent. - :rtype: ZfsDataset | None - """ - if self.is_snapshot: - return self.zfs_node.get_dataset(self.prefix) - else: - stripped = self.rstrip_path(1) - if stripped: - return self.zfs_node.get_dataset(stripped) - else: - return None - - def find_next_snapshot(self, snapshot_bookmark): - """find next snapshot in this dataset, according to snapshot or bookmark. None if it doesn't exist - - - - Args: - :type snapshot_bookmark: ZfsDataset - """ - - if not self.is_dataset: - raise (Exception("Please call this on a dataset.")) - - found = False - for snapshot in self.snapshots_bookmarks: - if snapshot == snapshot_bookmark: - found = True - else: - if found and snapshot.is_snapshot: - return snapshot - - return None - @property def exists_check(self): """check on disk if it exists""" @@ -347,45 +146,17 @@ class ZfsDataset: else: return self.exists_check - def create_filesystem(self, parents=False, unmountable=True): - """create a filesystem - - Args: - :type parents: bool - """ - - # recurse up - if parents and self.parent and not self.parent.exists: - self.parent.create_filesystem(parents, unmountable) - - cmd = ["zfs", "create"] - - if unmountable: - cmd.extend(["-o", "canmount=off"]) - - cmd.append(self.name) - self.zfs_node.run(cmd) - - self.force_exists = True - - def destroy(self, fail_exception=False, deferred=False, verbose=True): + def destroy(self, fail_exception=False, deferred=False): """destroy the dataset. by default failures are not an exception, so we can continue making backups Args: + :type deferred: bool :type fail_exception: bool """ - if verbose and not self.is_bookmark: - self.verbose("Destroying") - else: - self.debug("Destroying") - - if self.is_snapshot: - self.release() - try: - if deferred and self.is_snapshot: + if deferred: self.zfs_node.run(["zfs", "destroy", "-d", self.name]) else: self.zfs_node.run(["zfs", "destroy", self.name]) @@ -439,717 +210,6 @@ class ZfsDataset: """return true if this snapshot name belong to the current backup_name and snapshot formatting""" return self.timestamp is not None - @property - def holds(self): - """get list[holds] for dataset""" - - output = self.zfs_node.run(["zfs", "holds", "-H", self.name], valid_exitcodes=[0], tab_split=True, - readonly=True) - return map(lambda fields: fields[1], output) - - def is_hold(self): - """did we hold this snapshot?""" - return self.zfs_node.hold_name in self.holds - - def hold(self): - """hold dataset""" - self.debug("holding") - self.zfs_node.run(["zfs", "hold", self.zfs_node.hold_name, self.name], valid_exitcodes=[0, 1]) - - def release(self): - """release dataset""" - if self.zfs_node.readonly or self.is_hold(): - self.debug("releasing") - self.zfs_node.run(["zfs", "release", self.zfs_node.hold_name, self.name], valid_exitcodes=[0, 1]) - - @property - def timestamp(self): - """get timestamp from snapshot name. Only works for our own snapshots - with the correct format. Snapshots that are not ours always return None - - Note that the tag-part in the name is ignored, so snapsnots are ours regardless of their tag. - - :rtype: int|None - """ - - if self.zfs_node.tag_seperator in self.suffix: - tagless_suffix = self.suffix.split(self.zfs_node.tag_seperator)[0] - else: - tagless_suffix = self.suffix - - try: - dt = datetime.strptime(tagless_suffix, self.zfs_node.snapshot_time_format) - except ValueError: - return None - - if sys.version_info[0] >= 3: - from datetime import timezone - if self.zfs_node.utc: - dt = dt.replace(tzinfo=timezone.utc) - seconds = dt.timestamp() - else: - # python2 has no good functions to deal with UTC. Yet the unix timestamp - # must be in UTC to allow comparison against `time.time()` in on other parts - # of this project (e.g. Thinner.py). If we are handling UTC timestamps, - # we must adjust for that here. - if self.zfs_node.utc: - seconds = (dt - datetime(1970, 1, 1)).total_seconds() - else: - seconds = time.mktime(dt.timetuple()) - return seconds - - @property - def snapshots_bookmarks(self): - """get all snapshots and bookmarks of this dataset (ordered by createtxg, so its suitable to determine incremental zfs send order) - :rtype: list[ZfsDataset] - """ - - # cached? - if self.__snapshots_bookmarks is None: - self.debug("Getting snapshots and bookmarks") - - cmd = [ - "zfs", "list", "-d", "1", "-r", "-t", "snapshot,bookmark", "-H", "-o", "name", "-s", "createtxg", - self.name - ] - - self.__snapshots_bookmarks = self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), - force_exists=True) - - return self.__snapshots_bookmarks - - @property - def snapshots(self): - """get all snapshots of this dataset - :rtype: list[ZfsDataset] - """ - ret = [] - - for snapshot in self.snapshots_bookmarks: - if snapshot.is_snapshot: - ret.append(snapshot) - - return ret - - def cache_snapshot_bookmark(self, snapshot, force=False): - """Update our snapshot and bookmark cache (if we have any). Use force if you want to force the caching, potentially triggering a zfs list - Args: - :type snapshot: ZfsDataset - """ - - if force: - self.snapshots_bookmarks.append(snapshot) - - elif self.__snapshots_bookmarks is not None: - self.__snapshots_bookmarks.append(snapshot) - - @property - def our_snapshots(self): - """get list[snapshots] creates by us of this dataset""" - ret = [] - - for snapshot in self.snapshots: - if snapshot.is_ours: - ret.append(snapshot) - - return ret - - def find_snapshot_in_list(self, snapshots): - """return ZfsDataset from the list of snapshots, if it matches the snapshot_name. Otherwise None - Args: - :type snapshots: list[ZfsDataset] - :rtype: ZfsDataset|None - """ - - for snapshot in snapshots: - if snapshot.tagless_suffix == self.tagless_suffix: - return snapshot - - return None - - def find_guid_bookmark_snapshot(self, guid): - """find the first bookmark or snapshot that matches, prefers bookmarks. - Args: - :type guid:str - :rtype: ZfsDataset|None - """ - # Since this is slower, we only use it if the name matching with find_snapshot and find_bookmark doesn work. - - for bookmark in self.bookmarks: - if bookmark.properties['guid'] == guid: - return bookmark - - for snapshot in self.snapshots: - if snapshot.properties['guid'] == guid: - return snapshot - - return None - - def find_snapshot(self, snapshot): - """find snapshot by snapshot name (can be a suffix or a different - ZfsDataset) Returns None if it cant find it. - - Note that matches with our own snapshots will be done tagless. - - Args: - :rtype: ZfsDataset|None - :type snapshot: str|ZfsDataset|None - """ - - if snapshot is None: - return None - - if not isinstance(snapshot, ZfsDataset): - tagless_suffix = snapshot - else: - tagless_suffix = snapshot.tagless_suffix - - for snapshot in self.snapshots: - if snapshot.tagless_suffix == tagless_suffix: - return snapshot - - return None - - def find_bookmark(self, snapshot_bookmark, preferred_tag): - """find bookmark by bookmark name (can be a suffix or a different - ZfsDataset) Returns None if it cant find it. - - We try to find the bookmark with the preferred tag (which is usually a target path guid, to prevent conflicting bookmarks by multiple sends) - After that we return any bookmark that matches (and ignore the tag) - - Args: - :rtype: ZfsDataset|None - :type snapshot_bookmark: str|ZfsDataset|None - :type ignore_tag: bool - """ - - if snapshot_bookmark is None: - return None - - if not isinstance(snapshot_bookmark, ZfsDataset): - tagless_suffix = snapshot_bookmark - else: - tagless_suffix = snapshot_bookmark.tagless_suffix - - for snapshot_bookmark in self.bookmarks: - if snapshot_bookmark.tagless_suffix == tagless_suffix and snapshot_bookmark.tag == preferred_tag: - return snapshot_bookmark - - for snapshot_bookmark in self.bookmarks: - if snapshot_bookmark.tagless_suffix == tagless_suffix: - return snapshot_bookmark - - return None - - def find_exact_bookmark(self, bookmark_name): - """find exact bookmark name, or retruns none - - :rtype: ZfsDataset|None - """ - - for snapshot_bookmark in self.bookmarks: - if snapshot_bookmark.name == bookmark_name: - return snapshot_bookmark - return None - - def find_snapshot_index(self, snapshot): - """find exact snapshot index by snapshot (can be a snapshot_name or - ZfsDataset) - - Args: - :type snapshot: str or ZfsDataset - """ - - if not isinstance(snapshot, ZfsDataset): - snapshot_name = snapshot - else: - snapshot_name = snapshot.suffix - - index = 0 - for snapshot in self.snapshots: - if snapshot.suffix == snapshot_name: - return index - index = index + 1 - - return None - - @property - def written_since_ours(self): - """get number of bytes written since our last snapshot""" - - if self.__written_since_ours is None: - latest_snapshot = self.our_snapshots[-1] - - self.debug("Getting bytes written since our last snapshot") - cmd = ["zfs", "get", "-H", "-ovalue", "-p", "written@" + str(latest_snapshot), self.name] - - output = self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0]) - - self.__written_since_ours = int(output[0]) - - return self.__written_since_ours - - def is_changed_ours(self, min_changed_bytes=1): - """dataset is changed since OUR latest snapshot? - - Args: - :type min_changed_bytes: int - """ - - if min_changed_bytes == 0: - return True - - if not self.our_snapshots: - return True - - # NOTE: filesystems can have a very small amount written without actual changes in some cases - if self.written_since_ours < min_changed_bytes: - return False - - return True - - def bookmark(self, tag): - """Bookmark this snapshot, and return the bookmark. If the bookmark already exist, it returns it.""" - # NOTE: we use the tag to add the target_path GUID, so that we can have multiple bookmarks for the same snapshot, but for different targets. - # This is to make sure you can send a backup to two locations, without them interfering with eachothers bookmarks. - - if not self.is_snapshot: - raise (Exception("Can only bookmark a snapshot!")) - - bookmark_name = self.prefix + "#" + self.tagless_suffix + self.zfs_node.tag_seperator + tag - - # does this exact bookmark (including tag) already exists? - existing_bookmark = self.parent.find_exact_bookmark(bookmark_name) - if existing_bookmark is not None: - return existing_bookmark - - self.debug("Bookmarking {}".format(bookmark_name)) - - cmd = [ - "zfs", "bookmark", self.name, bookmark_name - ] - - self.zfs_node.run(cmd=cmd) - - bookmark = self.zfs_node.get_dataset(bookmark_name, force_exists=True) - self.cache_snapshot_bookmark(bookmark) - return bookmark - - @property - def bookmarks(self): - - ret = [] - - for bookmark in self.snapshots_bookmarks: - if bookmark.is_bookmark: - ret.append(bookmark) - - return ret - - @property - def recursive_datasets(self, types="filesystem,volume"): - """get all (non-snapshot) datasets recursively under us - - Args: - :type types: str - :rtype: list[ZfsDataset] - """ - - if self.__recursive_datasets is None: - self.debug("Getting all recursive datasets under us") - - names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ - "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name - ]) - - self.__recursive_datasets = self.zfs_node.get_datasets(names[1:], force_exists=True) - - return self.__recursive_datasets - - @property - def datasets(self, types="filesystem,volume"): - """get all (non-snapshot) datasets directly under us - - Args: - :type types: str - """ - - if self.__datasets is None: - self.debug("Getting all datasets under us") - - names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ - "zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name - ]) - - self.__datasets = self.zfs_node.get_datasets(names[1:], force_exists=True) - - return self.__datasets - - def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, write_embedded, - send_pipes, zfs_compressed): - """returns a pipe with zfs send output for this snapshot - - resume_token: resume sending from this token. (in that case we don't - need to know snapshot names) - - Args: - :param send_pipes: output cmd array that will be added to actual zfs send command. (e.g. mbuffer or compression program) - :type send_pipes: list[str] - :type features: list[str] - :type prev_snapshot: ZfsDataset - :type resume_token: str - :type show_progress: bool - :type raw: bool - """ - # build source command - cmd = [] - - cmd.extend(["zfs", "send", ]) - - # all kind of performance options: - if 'large_blocks' in features and "-L" in self.zfs_node.supported_send_options: - # large block support (only if recordsize>128k which is seldomly used) - cmd.append("-L") # --large-block - - if write_embedded and 'embedded_data' in features and "-e" in self.zfs_node.supported_send_options: - cmd.append("-e") # --embed; WRITE_EMBEDDED, more compact stream - - if zfs_compressed and "-c" in self.zfs_node.supported_send_options: - cmd.append("-c") # --compressed; use compressed WRITE records - - # raw? (send over encrypted data in its original encrypted form without decrypting) - if raw: - cmd.append("--raw") - - # progress output - if show_progress: - cmd.append("-v") # --verbose - cmd.append("-P") # --parsable - - # resume a previous send? (don't need more parameters in that case) - if resume_token: - cmd.extend(["-t", resume_token]) - - else: - # send properties - if send_properties: - cmd.append("-p") # --props - - # incremental? - if prev_snapshot: - cmd.extend(["-i", prev_snapshot.typed_suffix]) - - cmd.append(self.name) - - cmd.extend(send_pipes) - - output_pipe = self.zfs_node.run(cmd, pipe=True, readonly=True) - - return output_pipe - - def recv_pipe(self, pipe, features, recv_pipes, filter_properties=None, set_properties=None, ignore_exit_code=False, - force=False): - """starts a zfs recv for this snapshot and uses pipe as input - - note: you can it both on a snapshot or filesystem object. The - resulting zfs command is the same, only our object cache is invalidated - differently. - - Args: - :param recv_pipes: input cmd array that will be prepended to actual zfs recv command. (e.g. mbuffer or decompression program) - :type pipe: subprocess.pOpen - :type features: list[str] - :type filter_properties: list[str] - :type set_properties: list[str] - :type ignore_exit_code: bool - """ - - if set_properties is None: - set_properties = [] - - if filter_properties is None: - filter_properties = [] - - # build target command - cmd = [] - - cmd.extend(recv_pipes) - - cmd.extend(["zfs", "recv"]) - - # don't let zfs recv mount everything thats received (even with canmount=noauto!) - cmd.append("-u") - - for property_ in filter_properties: - cmd.extend(["-x", property_]) - - for property_ in set_properties: - cmd.extend(["-o", property_]) - - # verbose output - cmd.append("-v") - - if force: - cmd.append("-F") - - if 'extensible_dataset' in features and "-s" in self.zfs_node.supported_recv_options: - # support resuming - self.debug("Enabled resume support") - cmd.append("-s") - - cmd.append(self.prefix) - - if ignore_exit_code: - valid_exitcodes = [] - else: - valid_exitcodes = [0] - - # self.zfs_node.reset_progress() - self.zfs_node.run(cmd, inp=pipe, valid_exitcodes=valid_exitcodes) - - # invalidate cache - self.invalidate_cache() - - # in test mode we assume everything was ok and it exists - if self.zfs_node.readonly: - self.force_exists = True - - # check if transfer was really ok (exit codes have been wrong before due to bugs in zfs-utils and some - # errors should be ignored, thats where the ignore_exitcodes is for.) - if not self.exists: - self.error("error during transfer") - raise (Exception("Target doesn't exist after transfer, something went wrong.")) - - # at this point we're sure the actual dataset exists - self.parent.force_exists = True - - def automount(self): - """Mount the dataset as if one did a zfs mount -a, but only for this dataset - Failure to mount doesnt result in an exception, but outputs errors to STDERR. - - """ - - self.debug("Auto mounting") - - if self.properties['type'] != "filesystem": - return - - if self.properties['canmount'] != 'on': - return - - if self.properties['mountpoint'] == 'legacy': - return - - if self.properties['mountpoint'] == 'none': - return - - if self.properties['encryption'] != 'off' and self.properties['keystatus'] == 'unavailable': - return - - self.zfs_node.run(["zfs", "mount", self.name], valid_exitcodes=[0, 1]) - - def transfer_snapshot(self, target_snapshot, features, prev_snapshot, show_progress, - filter_properties, set_properties, ignore_recv_exit_code, resume_token, - raw, send_properties, write_embedded, send_pipes, recv_pipes, zfs_compressed, force): - """transfer this snapshot to target_snapshot. specify prev_snapshot for - incremental transfer - - connects a send_pipe() to recv_pipe() - - Args: - :type send_pipes: list[str] - :type recv_pipes: list[str] - :type target_snapshot: ZfsDataset - :type features: list[str] - :type prev_snapshot: ZfsDataset - :type show_progress: bool - :type filter_properties: list[str] - :type set_properties: list[str] - :type ignore_recv_exit_code: bool - :type resume_token: str - :type raw: bool - """ - - if set_properties is None: - set_properties = [] - if filter_properties is None: - filter_properties = [] - - self.debug("Transfer snapshot to {}".format(target_snapshot.prefix)) - - if resume_token: - self.verbose("resuming") - - # initial or increment - if not prev_snapshot: - self.verbose("-> {} (new)".format(target_snapshot.prefix)) - else: - # incremental - self.verbose("-> {}".format(target_snapshot.prefix)) - - # do it - pipe = self.send_pipe(features=features, show_progress=show_progress, prev_snapshot=prev_snapshot, - resume_token=resume_token, raw=raw, send_properties=send_properties, - write_embedded=write_embedded, send_pipes=send_pipes, zfs_compressed=zfs_compressed) - target_snapshot.recv_pipe(pipe, features=features, filter_properties=filter_properties, - set_properties=set_properties, ignore_exit_code=ignore_recv_exit_code, - recv_pipes=recv_pipes, force=force) - - # try to automount it, if its the initial transfer - if not prev_snapshot: - # in test mode it doesnt actually exist, so dont try to mount it/read properties - if not target_snapshot.zfs_node.readonly: - target_snapshot.parent.automount() - - def abort_resume(self): - """abort current resume state""" - self.debug("Aborting resume") - self.zfs_node.run(["zfs", "recv", "-A", self.name]) - - def rollback(self): - """rollback to latest existing snapshot on this dataset""" - - for snapshot in reversed(self.snapshots): - if snapshot.exists: - self.debug("Rolling back") - self.zfs_node.run(["zfs", "rollback", snapshot.name]) - return - - def get_resume_snapshot(self, resume_token): - """returns snapshot that will be resumed by this resume token (run this - on source with target-token) - - Args: - :type resume_token: str - :rtype: ZfsDataset|None - """ - # use zfs send -n option to determine this - # NOTE: on smartos stderr, on linux stdout - (stdout, stderr) = self.zfs_node.run(["zfs", "send", "-t", resume_token, "-n", "-v"], valid_exitcodes=[0, 255], - readonly=True, return_stderr=True) - if stdout: - lines = stdout - else: - lines = stderr - for line in lines: - matches = re.findall("toname = .*@(.*)", line) - if matches: - snapshot_name = matches[0] - snapshot = self.zfs_node.get_dataset(self.prefix + "@" + snapshot_name) - snapshot.debug("resume token belongs to this snapshot") - return snapshot - - return None - - def thin_list(self, keeps=None, ignores=None): - """determines list[snapshots] that should be kept or deleted based on - the thinning schedule. cull the herd! - - returns: ( keeps, obsoletes ) - - Args: - :param keeps: list[snapshots] to always keep (usually the last) - :param ignores: snapshots to completely ignore (usually incompatible target snapshots that are going to be destroyed anyway) - :type keeps: list[ZfsDataset] - :type ignores: list[ZfsDataset] - """ - - if ignores is None: - ignores = [] - if keeps is None: - keeps = [] - - snapshots = [snapshot for snapshot in self.our_snapshots if snapshot not in ignores] - - return self.zfs_node.thin_list(snapshots, keep_snapshots=keeps) - - def thin(self, skip_holds=False): - """destroys snapshots according to thin_list, except last snapshot - - Args: - :type skip_holds: bool - """ - - (keeps, obsoletes) = self.thin_list(keeps=self.our_snapshots[-1:]) - for obsolete in obsoletes: - if skip_holds and obsolete.is_hold(): - obsolete.verbose("Keeping (common snapshot)") - else: - obsolete.destroy() - self.snapshots.remove(obsolete) - - def find_common_snapshot(self, target_dataset, guid_check, bookmark_tag): - """find latest common snapshot/bookmark between us and target returns None if its - an initial transfer. - - On the source it prefers the specified bookmark_name - - Args: - :rtype: tuple[ZfsDataset, ZfsDataset] | tuple[None,None] - :type guid_check: bool - :type target_dataset: ZfsDataset - :type bookmark_tag: str - """ - - if not target_dataset.exists or not target_dataset.snapshots: - # target has nothing yet - return None, None - else: - for target_snapshot in reversed(target_dataset.snapshots): - - # Prefer bookmarks to snapshots - source_bookmark = self.find_bookmark(target_snapshot, preferred_tag=bookmark_tag) - if source_bookmark: - if guid_check and source_bookmark.properties['guid'] != target_snapshot.properties['guid']: - source_bookmark.warning("Bookmark has mismatching GUID, ignoring.") - else: - source_bookmark.debug("Common bookmark") - return source_bookmark, target_snapshot - - # Source snapshot with same suffix? - source_snapshot = self.find_snapshot(target_snapshot) - if source_snapshot: - if guid_check and source_snapshot.properties['guid'] != target_snapshot.properties['guid']: - source_snapshot.warning("Snapshot has mismatching GUID, ignoring.") - else: - source_snapshot.debug("Common snapshot") - return source_snapshot, target_snapshot - - # Extensive GUID search (slower but works with all names) - try: - source_bookmark_snapshot = self.find_guid_bookmark_snapshot(target_snapshot.properties['guid']) - if source_bookmark_snapshot is not None: - return source_bookmark_snapshot, target_snapshot - except ExecuteError as e: - # in readonly mode we igore a failed property read for non existing snapshots - if not self.zfs_node.readonly: - raise e - - raise (Exception("Cant find common bookmark or snapshot with target.")) - - def find_incompatible_snapshots(self, target_common_snapshot, raw): - """returns a list[snapshots] that is incompatible for a zfs recv onto - the common_snapshot. all direct followup snapshots with written=0 are - compatible. - - in raw-mode nothing is compatible. issue #219 - - Args: - :type target_common_snapshot: ZfsDataset - :type raw: bool - """ - - ret = [] - - if target_common_snapshot and self.snapshots: - followup = True - for snapshot in self.snapshots[self.find_snapshot_index(target_common_snapshot) + 1:]: - if raw or not followup or int(snapshot.properties['written']) != 0: - followup = False - ret.append(snapshot) - - return ret - def get_allowed_properties(self, filter_properties, set_properties): """only returns lists of allowed properties for this dataset type @@ -1172,340 +232,6 @@ class ZfsDataset: return allowed_filter_properties, allowed_set_properties - def _pre_clean(self, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): - """cleanup old stuff before starting snapshot syncing - - Args: - :type source_common_snapshot: ZfsDataset - :type target_dataset: ZfsDataset - :type source_obsoletes: list[ZfsDataset] - :type target_obsoletes: list[ZfsDataset] - :type target_transfers: list[ZfsDataset] - """ - - # on source: delete all obsoletes that are not in target_transfers (except common snapshot, if its not a bookmark) - for source_snapshot in self.snapshots: - if (source_snapshot in source_obsoletes - and source_common_snapshot != source_snapshot - and source_snapshot.find_snapshot_in_list(target_transfers) is None): - source_snapshot.destroy() - - # on target: destroy everything thats obsolete, except common_snapshot - if target_dataset.exists: - for target_snapshot in target_dataset.snapshots: - if (target_snapshot in target_obsoletes) \ - and (not source_common_snapshot or ( - target_snapshot.tagless_suffix != source_common_snapshot.tagless_suffix)): - if target_snapshot.exists: - target_snapshot.destroy() - - def _validate_resume_token(self, target_dataset, start_snapshot): - """validate and get (or destory) resume token - - Args: - :type target_dataset: ZfsDataset - :type start_snapshot: ZfsDataset - """ - - if target_dataset.exists and 'receive_resume_token' in target_dataset.properties: - if start_snapshot is None: - target_dataset.verbose("Aborting resume, its obsolete.") - target_dataset.abort_resume() - else: - resume_token = target_dataset.properties['receive_resume_token'] - # not valid anymore - resume_snapshot = self.get_resume_snapshot(resume_token) - if not resume_snapshot or start_snapshot.suffix != resume_snapshot.suffix: - target_dataset.verbose("Aborting resume, its no longer valid.") - target_dataset.abort_resume() - else: - return resume_token - - def _plan_sync(self, target_dataset, also_other_snapshots, guid_check, raw, bookmark_tag): - """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. - - Args: - :rtype: ( ZfsDataset, list[ZfsDataset], list[ZfsDataset], list[ZfsDataset], list[ZfsDataset] ) - :type target_dataset: ZfsDataset - :type also_other_snapshots: bool - :type guid_check: bool - :type raw: bool - :type bookmark_tag: str - - Returns: - tuple: A tuple containing: - - ZfsDataset: The common snapshot - - list[ZfsDataset]: Our obsolete source snapshots, after transfer is done. (will be thinned asap) - - list[ZfsDataset]: Our obsolete target snapshots, after transfer is done. (will be thinned asap) - - list[ZfsDataset]: Transfer target snapshots. These need to be transferred. - - list[ZfsDataset]: Incompatible target snapshots. Target snapshots that are in the way, after the common snapshot. (need to be destroyed to continue) - - - """ - - ### 1: determine common and start snapshot - - target_dataset.debug("Determining start snapshot") - (source_common_snapshot, target_common_snapshot) = self.find_common_snapshot(target_dataset, - guid_check=guid_check, - bookmark_tag=bookmark_tag) - # if source_common_snapshot: - # source_common_snapshot.verbose("Common snapshot or bookmark") - incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(target_common_snapshot, raw) - - # let thinner decide whats obsolete on source after the transfer is done - source_obsoletes = [] - if self.our_snapshots: - source_obsoletes = self.thin_list()[1] - - ### 2: Determine possible target snapshots - - # start with snapshots that already exist, minus imcompatibles - if target_dataset.exists: - possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots if - snapshot not in incompatible_target_snapshots] - else: - possible_target_snapshots = [] - - # add all snapshots from the source, starting after the common snapshot if it exists - if source_common_snapshot: - source_snapshot = self.find_next_snapshot(source_common_snapshot) - else: - if self.snapshots: - source_snapshot = self.snapshots[0] - else: - source_snapshot = None - - while source_snapshot: - # we want it? - if (also_other_snapshots or source_snapshot.is_ours) and not source_snapshot.is_snapshot_excluded: - # create virtual target snapshot - target_snapshot = target_dataset.zfs_node.get_dataset( - target_dataset.prefix + source_snapshot.typed_suffix, force_exists=False) - possible_target_snapshots.append(target_snapshot) - source_snapshot = self.find_next_snapshot(source_snapshot) - - ### 3: Let the thinner decide what it wants by looking at all the possible target_snaphots at once. - # always keep the last target snapshot as common snapshot. - if possible_target_snapshots: - (target_keeps, target_obsoletes) = target_dataset.zfs_node.thin_list(possible_target_snapshots, - keep_snapshots=[ - possible_target_snapshots[-1]]) - else: - target_keeps = [] - target_obsoletes = [] - - ### 4: Look at what the thinner wants and create a list of snapshots we still need to transfer - target_transfers = [] - for target_keep in target_keeps: - if not target_keep.exists: - target_transfers.append(target_keep) - - return source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots - - def handle_incompatible_snapshots(self, incompatible_target_snapshots, destroy_incompatible): - """destroy incompatbile snapshots on target before sync, or inform user - what to do - - Args: - :type incompatible_target_snapshots: list[ZfsDataset] - :type destroy_incompatible: bool - """ - - if incompatible_target_snapshots: - if not destroy_incompatible: - for snapshot in incompatible_target_snapshots: - snapshot.error("Incompatible snapshot") - raise (Exception("Please destroy incompatible snapshots on target, or use --destroy-incompatible.")) - else: - for snapshot in incompatible_target_snapshots: - snapshot.verbose("Incompatible snapshot") - snapshot.destroy(fail_exception=True) - self.snapshots.remove(snapshot) - - if len(incompatible_target_snapshots) > 0: - self.rollback() - - def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, - ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, - no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, - use_bookmarks, bookmark_tag): - """sync this dataset's snapshots to target_dataset, while also thinning - out old snapshots along the way. - - Args: - :type send_pipes: list[str] - :type recv_pipes: list[str] - :type target_dataset: ZfsDataset - :type features: list[str] - :type show_progress: bool - :type filter_properties: list[str] - :type set_properties: list[str] - :type ignore_recv_exit_code: bool - :type holds: bool - :type rollback: bool - :type decrypt: bool - :type also_other_snapshots: bool - :type no_send: bool - :type guid_check: bool - :type use_bookmarks: bool - :type bookmark_tag: str - """ - - # self.verbose("-> {}".format(target_dataset)) - - # defaults for these settings if there is no encryption stuff going on: - send_properties = True - raw = False - write_embedded = True - - # source dataset encrypted? - if self.properties.get('encryption', 'off') != 'off': - # user wants to send it over decrypted? - if decrypt: - # when decrypting, zfs cant send properties - send_properties = False - else: - # keep data encrypted by sending it raw (including properties) - raw = True - - (source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, - incompatible_target_snapshots) = \ - self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, - guid_check=guid_check, raw=raw, bookmark_tag=bookmark_tag) - - # NOTE: we do a pre-clean because we dont want filesystems to fillup when backups keep failing. - # Also usefull with no_send to still cleanup stuff. - self._pre_clean( - source_common_snapshot=source_common_snapshot, target_dataset=target_dataset, - target_transfers=target_transfers, target_obsoletes=target_obsoletes, source_obsoletes=source_obsoletes) - - # handle incompatible stuff on target - target_dataset.handle_incompatible_snapshots(incompatible_target_snapshots, destroy_incompatible) - - # now actually transfer the snapshots, if we want - if no_send or len(target_transfers) == 0: - return - - # check if we can resume - resume_token = self._validate_resume_token(target_dataset, target_transfers[0]) - - (active_filter_properties, active_set_properties) = self.get_allowed_properties(filter_properties, - set_properties) - - # always filter properties that start with 'autobackup:' (https://github.com/psy0rz/zfs_autobackup/issues/221) - for property in self.properties: - if property.startswith('autobackup:'): - active_filter_properties.append(property) - - # encrypt at target? - if encrypt and not raw: - # filter out encryption properties to let encryption on the target take place - active_filter_properties.extend(["keylocation", "pbkdf2iters", "keyformat", "encryption"]) - write_embedded = False - - # now actually transfer the snapshots - do_rollback = rollback - prev_source_snapshot_bookmark = source_common_snapshot - prev_target_snapshot = target_dataset.find_snapshot(source_common_snapshot) - for target_snapshot in target_transfers: - - source_snapshot = self.find_snapshot(target_snapshot) - - # do the rollback, one time at first transfer - if do_rollback: - target_dataset.rollback() - do_rollback = False - - source_snapshot.transfer_snapshot(target_snapshot, features=features, - prev_snapshot=prev_source_snapshot_bookmark, show_progress=show_progress, - filter_properties=active_filter_properties, - set_properties=active_set_properties, - ignore_recv_exit_code=ignore_recv_exit_code, - resume_token=resume_token, write_embedded=write_embedded, raw=raw, - send_properties=send_properties, send_pipes=send_pipes, - recv_pipes=recv_pipes, zfs_compressed=zfs_compressed, force=force) - - resume_token = None - - # hold/release common snapshot on the target. - if holds: - target_snapshot.hold() - - if prev_target_snapshot: - prev_target_snapshot.release() - - if use_bookmarks: - # bookmark common snapshot, and clean up obsolete snapshots and bookmark - source_bookmark = source_snapshot.bookmark(bookmark_tag) - if source_snapshot in source_obsoletes: - source_snapshot.destroy() - - # TODO: make a better is_ours specially for bookmarks, as part of the next refactoring splitting in more classes - # delete any bookmark that ends in ours tag_seprator + tag. - if prev_source_snapshot_bookmark and prev_source_snapshot_bookmark.is_bookmark and prev_source_snapshot_bookmark.name.endswith( - self.zfs_node.tag_seperator + bookmark_tag): - prev_source_snapshot_bookmark.destroy() - - - # dont use bookmarks - else: - source_bookmark = None - if holds: - source_snapshot.hold() - - # release hold, cleanup obsolete snapshot - if prev_source_snapshot_bookmark and prev_source_snapshot_bookmark.is_snapshot: - prev_source_snapshot_bookmark.release() - if prev_source_snapshot_bookmark in source_obsoletes: - prev_source_snapshot_bookmark.destroy() - - # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, - # the rest was already destroyed or will not be send) - if prev_target_snapshot in target_obsoletes: - prev_target_snapshot.destroy() - - # we always try to use the bookmark during incremental send - if source_bookmark: - prev_source_snapshot_bookmark = source_bookmark - else: - prev_source_snapshot_bookmark = source_snapshot - - prev_target_snapshot = target_snapshot - - def mount(self, mount_point): - - self.debug("Mounting") - - cmd = [ - "mount", "-tzfs", self.name, mount_point - ] - - self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - - def unmount(self, mount_point): - - self.debug("Unmounting") - - cmd = [ - "umount", mount_point - ] - - self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - - def clone(self, name): - """clones this snapshot and returns ZfsDataset of the clone""" - - self.debug("Cloning to {}".format(name)) - - cmd = [ - "zfs", "clone", self.name, name - ] - - self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - - return self.zfs_node.get_dataset(name, force_exists=True) - def set(self, prop, value): """set a zfs property""" diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index 7d91c68..27cd2b2 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -133,7 +133,18 @@ class ZfsNode(ExecuteNode): :rtype: ZfsDataset """ - return self.__datasets.setdefault(name, ZfsDataset(self, name, force_exists)) + if name in self.__datasets: + return self.__datasets[name] + + if '@' in name: + + self.__datasets[name] = ZfsDataset(self, name, force_exists=force_exists) + elif '#' in name: + + self.__datasets[name] = ZfsDataset(self, name, force_exists=force_exists) + else: + + self.__datasets[name] = ZfsDataset(self, name, force_exists=force_exists) def get_datasets(self, names, force_exists=None): """get a list of ZfsDataset() object from names. stores objects internally to enable caching @@ -319,3 +330,28 @@ class ZfsNode(ExecuteNode): # returns None when no property is set. return (selected_filesystems, excluded_filesystems) + + def get_resume_snapshot(self, resume_token): + """returns snapshot that will be resumed by this resume token (run this + on source node with target-token) + + Args: + :type resume_token: str + :rtype: ZfsDataset|None + """ + # use zfs send -n option to determine this + # NOTE: on smartos stderr, on linux stdout + (stdout, stderr) = self.run(["zfs", "send", "-t", resume_token, "-n", "-v"], valid_exitcodes=[0, 255], + readonly=True, return_stderr=True) + if stdout: + lines = stdout + else: + lines = stderr + for line in lines: + matches = re.findall("toname = (.*@.*)", line) + if matches: + snapshot = self.get_dataset(matches[0]) + snapshot.debug("resume token belongs to this snapshot") + return snapshot + + return None diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py new file mode 100644 index 0000000..4a245d2 --- /dev/null +++ b/zfs_autobackup/ZfsSnapshot.py @@ -0,0 +1,354 @@ +from .ZfsDataset import ZfsDataset +from .ExecuteNode import ExecuteError + + +class ZfsSnapshot(ZfsDataset): + """A ZFS snapshot""" + + @property + def prefix(self): + (filesystem, snapshot) = self.name.split("@") + return filesystem + + @property + def suffix(self): + (filesystem, snapshot_name) = self.name.split("@") + return snapshot_name + + @property + def typed_suffix(self): + return "@" + self.suffix + + @property + def tagless_suffix(self): + """snapshot or bookmark part of the name, but without the tag. (if its our snapshot we remove the tag)""" + + if self.is_ours and self.zfs_node.tag_seperator in self.suffix: + return self.suffix.split(self.zfs_node.tag_seperator)[0] + else: + return self.suffix + + @property + def tag(self): + """the tag-part of a snapshot or bookmark, if any. can return None""" + + if self.is_ours and self.zfs_node.tag_seperator in self.suffix: + return self.suffix.split(self.zfs_node.tag_seperator)[1] + else: + return None + + @property + def is_snapshot_excluded(self): + """true if this dataset is a snapshot and matches the exclude pattern""" + suffix = self.suffix + for pattern in self.zfs_node.exclude_snapshot_patterns: + if pattern.search(suffix) is not None: + self.debug("Excluded (path matches snapshot exclude pattern)") + return True + + def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, write_embedded, + send_pipes, zfs_compressed): + """returns a pipe with zfs send output for this snapshot + + resume_token: resume sending from this token. (in that case we don't + need to know snapshot names) + + Args: + :param send_pipes: output cmd array that will be added to actual zfs send command. (e.g. mbuffer or compression program) + :type send_pipes: list[str] + :type features: list[str] + :type prev_snapshot: ZfsDataset + :type resume_token: str + :type show_progress: bool + :type raw: bool + """ + # build source command + cmd = [] + + cmd.extend(["zfs", "send", ]) + + # all kind of performance options: + if 'large_blocks' in features and "-L" in self.zfs_node.supported_send_options: + # large block support (only if recordsize>128k which is seldomly used) + cmd.append("-L") # --large-block + + if write_embedded and 'embedded_data' in features and "-e" in self.zfs_node.supported_send_options: + cmd.append("-e") # --embed; WRITE_EMBEDDED, more compact stream + + if zfs_compressed and "-c" in self.zfs_node.supported_send_options: + cmd.append("-c") # --compressed; use compressed WRITE records + + # raw? (send over encrypted data in its original encrypted form without decrypting) + if raw: + cmd.append("--raw") + + # progress output + if show_progress: + cmd.append("-v") # --verbose + cmd.append("-P") # --parsable + + # resume a previous send? (don't need more parameters in that case) + if resume_token: + cmd.extend(["-t", resume_token]) + + else: + # send properties + if send_properties: + cmd.append("-p") # --props + + # incremental? + if prev_snapshot: + cmd.extend(["-i", prev_snapshot.typed_suffix]) + + cmd.append(self.name) + + cmd.extend(send_pipes) + + output_pipe = self.zfs_node.run(cmd, pipe=True, readonly=True) + + return output_pipe + + def recv_pipe(self, pipe, features, recv_pipes, filter_properties=None, set_properties=None, ignore_exit_code=False, + force=False): + """starts a zfs recv for this snapshot and uses pipe as input + + note: you can it both on a snapshot or filesystem object. The + resulting zfs command is the same, only our object cache is invalidated + differently. + + Args: + :param recv_pipes: input cmd array that will be prepended to actual zfs recv command. (e.g. mbuffer or decompression program) + :type pipe: subprocess.pOpen + :type features: list[str] + :type filter_properties: list[str] + :type set_properties: list[str] + :type ignore_exit_code: bool + """ + + if set_properties is None: + set_properties = [] + + if filter_properties is None: + filter_properties = [] + + # build target command + cmd = [] + + cmd.extend(recv_pipes) + + cmd.extend(["zfs", "recv"]) + + # don't let zfs recv mount everything thats received (even with canmount=noauto!) + cmd.append("-u") + + for property_ in filter_properties: + cmd.extend(["-x", property_]) + + for property_ in set_properties: + cmd.extend(["-o", property_]) + + # verbose output + cmd.append("-v") + + if force: + cmd.append("-F") + + if 'extensible_dataset' in features and "-s" in self.zfs_node.supported_recv_options: + # support resuming + self.debug("Enabled resume support") + cmd.append("-s") + + cmd.append(self.prefix) + + if ignore_exit_code: + valid_exitcodes = [] + else: + valid_exitcodes = [0] + + # self.zfs_node.reset_progress() + self.zfs_node.run(cmd, inp=pipe, valid_exitcodes=valid_exitcodes) + + # invalidate cache + self.invalidate_cache() + + # in test mode we assume everything was ok and it exists + if self.zfs_node.readonly: + self.force_exists = True + + # check if transfer was really ok (exit codes have been wrong before due to bugs in zfs-utils and some + # errors should be ignored, thats where the ignore_exitcodes is for.) + if not self.exists: + self.error("error during transfer") + raise (Exception("Target doesn't exist after transfer, something went wrong.")) + + # at this point we're sure the actual dataset exists + self.parent.force_exists = True + + def transfer_snapshot(self, target_snapshot, features, prev_snapshot, show_progress, + filter_properties, set_properties, ignore_recv_exit_code, resume_token, + raw, send_properties, write_embedded, send_pipes, recv_pipes, zfs_compressed, force): + """transfer this snapshot to target_snapshot. specify prev_snapshot for + incremental transfer + + connects a send_pipe() to recv_pipe() + + Args: + :type send_pipes: list[str] + :type recv_pipes: list[str] + :type target_snapshot: ZfsDataset + :type features: list[str] + :type prev_snapshot: ZfsDataset + :type show_progress: bool + :type filter_properties: list[str] + :type set_properties: list[str] + :type ignore_recv_exit_code: bool + :type resume_token: str + :type raw: bool + """ + + if set_properties is None: + set_properties = [] + if filter_properties is None: + filter_properties = [] + + self.debug("Transfer snapshot to {}".format(target_snapshot.prefix)) + + if resume_token: + self.verbose("resuming") + + # initial or increment + if not prev_snapshot: + self.verbose("-> {} (new)".format(target_snapshot.prefix)) + else: + # incremental + self.verbose("-> {}".format(target_snapshot.prefix)) + + # do it + pipe = self.send_pipe(features=features, show_progress=show_progress, prev_snapshot=prev_snapshot, + resume_token=resume_token, raw=raw, send_properties=send_properties, + write_embedded=write_embedded, send_pipes=send_pipes, zfs_compressed=zfs_compressed) + target_snapshot.recv_pipe(pipe, features=features, filter_properties=filter_properties, + set_properties=set_properties, ignore_exit_code=ignore_recv_exit_code, + recv_pipes=recv_pipes, force=force) + + # try to automount it, if its the initial transfer + if not prev_snapshot: + # in test mode it doesnt actually exist, so dont try to mount it/read properties + if not target_snapshot.zfs_node.readonly: + target_snapshot.parent.automount() + + def bookmark(self, tag): + """Bookmark this snapshot, and return the bookmark. If the bookmark already exist, it returns it.""" + # NOTE: we use the tag to add the target_path GUID, so that we can have multiple bookmarks for the same snapshot, but for different targets. + # This is to make sure you can send a backup to two locations, without them interfering with eachothers bookmarks. + + if not self.is_snapshot: + raise (Exception("Can only bookmark a snapshot!")) + + bookmark_name = self.prefix + "#" + self.tagless_suffix + self.zfs_node.tag_seperator + tag + + # does this exact bookmark (including tag) already exists? + existing_bookmark = self.parent.find_exact_bookmark(bookmark_name) + if existing_bookmark is not None: + return existing_bookmark + + self.debug("Bookmarking {}".format(bookmark_name)) + + cmd = [ + "zfs", "bookmark", self.name, bookmark_name + ] + + self.zfs_node.run(cmd=cmd) + + bookmark = self.zfs_node.get_dataset(bookmark_name, force_exists=True) + self.cache_snapshot_bookmark(bookmark) + return bookmark + + @property + def parent(self): + """get parent dataset + + :rtype: ZfsContainer | None + """ + return self.zfs_node.get_dataset(self.prefix) + + @property + def holds(self): + + output = self.zfs_node.run(["zfs", "holds", "-H", self.name], valid_exitcodes=[0], tab_split=True, + readonly=True) + return map(lambda fields: fields[1], output) + + def is_hold(self): + """did we hold this snapshot?""" + return self.zfs_node.hold_name in self.holds + + def hold(self): + """hold dataset""" + self.debug("holding") + self.zfs_node.run(["zfs", "hold", self.zfs_node.hold_name, self.name], valid_exitcodes=[0, 1]) + + def release(self): + """release dataset""" + if self.zfs_node.readonly or self.is_hold(): + self.debug("releasing") + self.zfs_node.run(["zfs", "release", self.zfs_node.hold_name, self.name], valid_exitcodes=[0, 1]) + + @property + def timestamp(self): + """get timestamp from snapshot name. Only works for our own snapshots + with the correct format. Snapshots that are not ours always return None + + Note that the tag-part in the name is ignored, so snapsnots are ours regardless of their tag. + + :rtype: int|None + """ + + if self.zfs_node.tag_seperator in self.suffix: + tagless_suffix = self.suffix.split(self.zfs_node.tag_seperator)[0] + else: + tagless_suffix = self.suffix + + try: + dt = datetime.strptime(tagless_suffix, self.zfs_node.snapshot_time_format) + except ValueError: + return None + + if sys.version_info[0] >= 3: + from datetime import timezone + if self.zfs_node.utc: + dt = dt.replace(tzinfo=timezone.utc) + seconds = dt.timestamp() + else: + # python2 has no good functions to deal with UTC. Yet the unix timestamp + # must be in UTC to allow comparison against `time.time()` in on other parts + # of this project (e.g. Thinner.py). If we are handling UTC timestamps, + # we must adjust for that here. + if self.zfs_node.utc: + seconds = (dt - datetime(1970, 1, 1)).total_seconds() + else: + seconds = time.mktime(dt.timetuple()) + return seconds + + def rollback(self): + + self.debug("Rolling back") + self.zfs_node.run(["zfs", "rollback", self.name]) + + def destroy(self, fail_exception=False, defered=False): + + self.verbose("Destroying") + return super().destroy(fail_exception=fail_exception, deferred=defered) + + def clone(self, name): + """clones this snapshot and returns ZfsDataset of the clone""" + + self.debug("Cloning to {}".format(name)) + + cmd = [ + "zfs", "clone", self.name, name + ] + + self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) + + return self.zfs_node.get_dataset(name, force_exists=True) From 1f60da1d659d4c390f0df8725caab4cf81658a0c Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 17 Jun 2025 23:52:35 +0200 Subject: [PATCH 061/135] started zfs-class splitting refactoring (everything broken from now on until its done) --- zfs_autobackup/ZfsBookmark.py | 4 ++++ zfs_autobackup/ZfsContainer.py | 20 ++++++++++++++++++++ zfs_autobackup/ZfsDataset.py | 28 ++++++---------------------- zfs_autobackup/ZfsNode.py | 25 ++++++++++++++----------- zfs_autobackup/ZfsSnapshot.py | 9 +++++++++ 5 files changed, 53 insertions(+), 33 deletions(-) diff --git a/zfs_autobackup/ZfsBookmark.py b/zfs_autobackup/ZfsBookmark.py index 4878665..ab4d042 100644 --- a/zfs_autobackup/ZfsBookmark.py +++ b/zfs_autobackup/ZfsBookmark.py @@ -1,5 +1,9 @@ from .ZfsDataset import ZfsDataset from .ExecuteNode import ExecuteError +import re +from datetime import datetime +import sys +import time class ZfsBookmark(ZfsDataset): diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 6a012dc..6d0824b 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -1,11 +1,31 @@ from .ZfsDataset import ZfsDataset from .ExecuteNode import ExecuteError from .ZfsSnapshot import ZfsSnapshot +import re +from datetime import datetime +import sys +import time class ZfsContainer(ZfsDataset): """Either a ZFS Filesystem or ZFS Dataset""" + def __init__(self, zfs_node, name, force_exists=None): + + super().__init__(zfs_node, name, force_exists=force_exists) + + self.__written_since_ours = None # type: None|int + self.__recursive_datasets = None # type: None|list[ZfsDataset] + self.__datasets = None # type: None|list[ZfsDataset] + self.__snapshots_bookmarks = None # type: None|list[ZfsDataset] + + def invalidate_cache(self): + super().invalidate_cache() + self.__written_since_ours = None + self.__recursive_datasets = None + self.__datasets = None + self.__snapshots_bookmarks = None + @property def snapshots(self): """get all snapshots of this dataset diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 9f08bb5..7c7921f 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1,8 +1,3 @@ -import re -from datetime import datetime -import sys -import time - from .ExecuteNode import ExecuteError @@ -25,32 +20,26 @@ class ZfsDataset: :type name: str :type force_exists: bool """ + + # prevent direct instantiation + if type(self) is ZfsDataset: + raise TypeError( + "ZfsDataset should instantiated directly. Use ZfsSnapshot, ZfsBookmark or ZfsContainer instead.") + self.zfs_node = zfs_node self.name = name # full actual name of dataset self.force_exists = force_exists # caching - # self.__snapshots = None # type: None|list[ZfsDataset] - self.__written_since_ours = None # type: None|int self.__exists_check = None # type: None|bool self.__properties = None # type: None|dict[str,str] - self.__recursive_datasets = None # type: None|list[ZfsDataset] - self.__datasets = None # type: None|list[ZfsDataset] - # self.__bookmarks = None # type: None|list[ZfsDataset] - self.__snapshots_bookmarks = None # type: None|list[ZfsDataset] def invalidate_cache(self): """clear caches""" self.force_exists = None - # self.__snapshots = None - self.__written_since_ours = None self.__exists_check = None self.__properties = None - self.__recursive_datasets = None - self.__datasets = None - # self.__bookmarks = None - self.__snapshots_bookmarks = None def __repr__(self): return "{}: {}".format(self.zfs_node, self.name) @@ -205,11 +194,6 @@ class ZfsDataset: else: return True - @property - def is_ours(self): - """return true if this snapshot name belong to the current backup_name and snapshot formatting""" - return self.timestamp is not None - def get_allowed_properties(self, filter_properties, set_properties): """only returns lists of allowed properties for this dataset type diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index 27cd2b2..cbdd5f7 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -2,14 +2,15 @@ from __future__ import print_function import re import shlex -import subprocess -import sys + import time from .ExecuteNode import ExecuteNode -from .Thinner import Thinner +from .ZfsContainer import ZfsContainer from .ZfsPool import ZfsPool from .ZfsDataset import ZfsDataset +from .ZfsSnapshot import ZfsSnapshot +from .ZfsBookmark import ZfsBookmark from .ExecuteNode import ExecuteError from .util import datetime_now @@ -59,8 +60,8 @@ class ZfsNode(ExecuteNode): self.__thinner = thinner # list of ZfsPools - self.__pools = {} - self.__datasets = {} + self.__pools = {} # type: dict[str, ZfsPool] + self.__datasets = {} # type: dict[str, ZfsContainer | ZfsBookmark | ZfsSnapshot] self._progress_total_bytes = 0 self._progress_start_time = time.time() @@ -130,7 +131,7 @@ class ZfsNode(ExecuteNode): def get_dataset(self, name, force_exists=None): """get a ZfsDataset() object from name. stores objects internally to enable caching :type name: str - :rtype: ZfsDataset + :rtype: ZfsContainer | ZfsBookmark | ZfsSnapshot """ if name in self.__datasets: @@ -138,17 +139,19 @@ class ZfsNode(ExecuteNode): if '@' in name: - self.__datasets[name] = ZfsDataset(self, name, force_exists=force_exists) + self.__datasets[name] = ZfsSnapshot(self, name, force_exists=force_exists) elif '#' in name: - self.__datasets[name] = ZfsDataset(self, name, force_exists=force_exists) + self.__datasets[name] = ZfsBookmark(self, name, force_exists=force_exists) else: - self.__datasets[name] = ZfsDataset(self, name, force_exists=force_exists) + self.__datasets[name] = ZfsContainer(self, name, force_exists=force_exists) + + return self.__datasets[name] def get_datasets(self, names, force_exists=None): """get a list of ZfsDataset() object from names. stores objects internally to enable caching - :rtype: list[ZfsDataset] + :rtype: list[ZfsContainer | ZfsBookmark | ZfsSnapshot] """ @@ -232,7 +235,7 @@ class ZfsNode(ExecuteNode): post_snapshot_cmds=[], set_snapshot_properties=[]): """create a consistent (atomic) snapshot of specified datasets, per pool. Args: - :type datasets: list[ZfsDataset] + :type datasets: list[ZfsContainer] """ pools = {} diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index 4a245d2..fac3ad6 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -1,5 +1,9 @@ from .ZfsDataset import ZfsDataset from .ExecuteNode import ExecuteError +import re +from datetime import datetime +import sys +import time class ZfsSnapshot(ZfsDataset): @@ -352,3 +356,8 @@ class ZfsSnapshot(ZfsDataset): self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) return self.zfs_node.get_dataset(name, force_exists=True) + + @property + def is_ours(self): + """return true if this snapshot name belong to the current backup_name and snapshot formatting""" + return self.timestamp is not None From 9e2432dd439042cf92502388950fac819a4b5935 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 18 Jun 2025 23:17:55 +0200 Subject: [PATCH 062/135] fixing and refactoring --- zfs_autobackup/ZfsBookmark.py | 30 ++--------- zfs_autobackup/ZfsCheck.py | 2 +- zfs_autobackup/ZfsDataset.py | 4 +- zfs_autobackup/ZfsPointInTime.py | 85 ++++++++++++++++++++++++++++++++ zfs_autobackup/ZfsSnapshot.py | 69 ++------------------------ 5 files changed, 96 insertions(+), 94 deletions(-) create mode 100644 zfs_autobackup/ZfsPointInTime.py diff --git a/zfs_autobackup/ZfsBookmark.py b/zfs_autobackup/ZfsBookmark.py index ab4d042..490c244 100644 --- a/zfs_autobackup/ZfsBookmark.py +++ b/zfs_autobackup/ZfsBookmark.py @@ -5,8 +5,10 @@ from datetime import datetime import sys import time +from .ZfsPointInTime import ZfsPointInTime -class ZfsBookmark(ZfsDataset): + +class ZfsBookmark(ZfsPointInTime): """A ZFS bookmark""" @property @@ -23,33 +25,9 @@ class ZfsBookmark(ZfsDataset): def typed_suffix(self): return "#" + self.suffix - @property - def tagless_suffix(self): - """snapshot or bookmark part of the name, but without the tag. (if its our snapshot we remove the tag)""" - if self.is_ours and self.zfs_node.tag_seperator in self.suffix: - return self.suffix.split(self.zfs_node.tag_seperator)[0] - else: - return self.suffix - @property - def tag(self): - """the tag-part of a snapshot or bookmark, if any. can return None""" - - if self.is_ours and self.zfs_node.tag_seperator in self.suffix: - return self.suffix.split(self.zfs_node.tag_seperator)[1] - else: - return None - - @property - def parent(self): - """get parent dataset - - :rtype: ZfsContainer | None - """ - return self.zfs_node.get_dataset(self.prefix) - - def destroy(self, fail_exception=False): + def destroy(self, fail_exception=False, **kwargs): self.debug("Destroying") return super().destroy(fail_exception=fail_exception) diff --git a/zfs_autobackup/ZfsCheck.py b/zfs_autobackup/ZfsCheck.py index b2a4ea9..0f410f8 100644 --- a/zfs_autobackup/ZfsCheck.py +++ b/zfs_autobackup/ZfsCheck.py @@ -117,7 +117,7 @@ class ZfsCheck(CliBase): """destroys temporary volume snapshot""" clone_name = get_tmp_clone_name(snapshot) clone = snapshot.zfs_node.get_dataset(clone_name) - clone.destroy(deferred=True, verbose=False) + clone.destroy() def generate_tree_hashes(self, prepared_target): diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 7c7921f..6350441 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1,4 +1,6 @@ from .ExecuteNode import ExecuteError +from .ZfsContainer import ZfsContainer +from .ZfsSnapshot import ZfsSnapshot class ZfsDataset: @@ -22,7 +24,7 @@ class ZfsDataset: """ # prevent direct instantiation - if type(self) is ZfsDataset: + if type(self) is not ZfsContainer and not ZfsSnapshot and not ZfsDataset: raise TypeError( "ZfsDataset should instantiated directly. Use ZfsSnapshot, ZfsBookmark or ZfsContainer instead.") diff --git a/zfs_autobackup/ZfsPointInTime.py b/zfs_autobackup/ZfsPointInTime.py new file mode 100644 index 0000000..26c3604 --- /dev/null +++ b/zfs_autobackup/ZfsPointInTime.py @@ -0,0 +1,85 @@ +from zfs_autobackup.ZfsDataset import ZfsDataset +from datetime import datetime + + +class ZfsPointInTime(ZfsDataset): + """contains stuff that is common to snapshots and bookmarks""" + + + #implemented in subclass + @property + def suffix(self): + return "" + + #implemented in subclass + @property + def prefix(self): + + return "" + @property + def timestamp(self): + """get timestamp from snapshot/bookmark name. Only works for our own snapshots + with the correct format. Snapshots that are not ours always return None + + Note that the tag-part in the name is ignored, so snapsnots are ours regardless of their tag. + + :rtype: int|None + """ + + if self.zfs_node.tag_seperator in self.suffix: + tagless_suffix = self.suffix.split(self.zfs_node.tag_seperator)[0] + else: + tagless_suffix = self.suffix + + try: + dt = datetime.strptime(tagless_suffix, self.zfs_node.snapshot_time_format) + except ValueError: + return None + + if sys.version_info[0] >= 3: + from datetime import timezone + if self.zfs_node.utc: + dt = dt.replace(tzinfo=timezone.utc) + seconds = dt.timestamp() + else: + # python2 has no good functions to deal with UTC. Yet the unix timestamp + # must be in UTC to allow comparison against `time.time()` in on other parts + # of this project (e.g. Thinner.py). If we are handling UTC timestamps, + # we must adjust for that here. + if self.zfs_node.utc: + seconds = (dt - datetime(1970, 1, 1)).total_seconds() + else: + seconds = time.mktime(dt.timetuple()) + return seconds + + + @property + def tagless_suffix(self): + """snapshot or bookmark part of the name, but without the tag. (if its our snapshot we remove the tag)""" + + if self.is_ours and self.zfs_node.tag_seperator in self.suffix: + return self.suffix.split(self.zfs_node.tag_seperator)[0] + else: + return self.suffix + + @property + def tag(self): + """the tag-part of a snapshot or bookmark, if any. can return None""" + + if self.is_ours and self.zfs_node.tag_seperator in self.suffix: + return self.suffix.split(self.zfs_node.tag_seperator)[1] + else: + return None + + @property + def is_ours(self): + """return true if this snapshot name belong to the current backup_name and snapshot formatting""" + return self.timestamp is not None + + @property + def parent(self): + """get parent dataset + + :rtype: ZfsContainer | None + """ + return self.zfs_node.get_dataset(self.prefix) diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index fac3ad6..e6a3e47 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -1,12 +1,13 @@ from .ZfsDataset import ZfsDataset from .ExecuteNode import ExecuteError import re -from datetime import datetime import sys import time +from .ZfsPointInTime import ZfsPointInTime -class ZfsSnapshot(ZfsDataset): + +class ZfsSnapshot(ZfsPointInTime): """A ZFS snapshot""" @property @@ -23,23 +24,6 @@ class ZfsSnapshot(ZfsDataset): def typed_suffix(self): return "@" + self.suffix - @property - def tagless_suffix(self): - """snapshot or bookmark part of the name, but without the tag. (if its our snapshot we remove the tag)""" - - if self.is_ours and self.zfs_node.tag_seperator in self.suffix: - return self.suffix.split(self.zfs_node.tag_seperator)[0] - else: - return self.suffix - - @property - def tag(self): - """the tag-part of a snapshot or bookmark, if any. can return None""" - - if self.is_ours and self.zfs_node.tag_seperator in self.suffix: - return self.suffix.split(self.zfs_node.tag_seperator)[1] - else: - return None @property def is_snapshot_excluded(self): @@ -268,14 +252,6 @@ class ZfsSnapshot(ZfsDataset): self.cache_snapshot_bookmark(bookmark) return bookmark - @property - def parent(self): - """get parent dataset - - :rtype: ZfsContainer | None - """ - return self.zfs_node.get_dataset(self.prefix) - @property def holds(self): @@ -298,41 +274,6 @@ class ZfsSnapshot(ZfsDataset): self.debug("releasing") self.zfs_node.run(["zfs", "release", self.zfs_node.hold_name, self.name], valid_exitcodes=[0, 1]) - @property - def timestamp(self): - """get timestamp from snapshot name. Only works for our own snapshots - with the correct format. Snapshots that are not ours always return None - - Note that the tag-part in the name is ignored, so snapsnots are ours regardless of their tag. - - :rtype: int|None - """ - - if self.zfs_node.tag_seperator in self.suffix: - tagless_suffix = self.suffix.split(self.zfs_node.tag_seperator)[0] - else: - tagless_suffix = self.suffix - - try: - dt = datetime.strptime(tagless_suffix, self.zfs_node.snapshot_time_format) - except ValueError: - return None - - if sys.version_info[0] >= 3: - from datetime import timezone - if self.zfs_node.utc: - dt = dt.replace(tzinfo=timezone.utc) - seconds = dt.timestamp() - else: - # python2 has no good functions to deal with UTC. Yet the unix timestamp - # must be in UTC to allow comparison against `time.time()` in on other parts - # of this project (e.g. Thinner.py). If we are handling UTC timestamps, - # we must adjust for that here. - if self.zfs_node.utc: - seconds = (dt - datetime(1970, 1, 1)).total_seconds() - else: - seconds = time.mktime(dt.timetuple()) - return seconds def rollback(self): @@ -357,7 +298,3 @@ class ZfsSnapshot(ZfsDataset): return self.zfs_node.get_dataset(name, force_exists=True) - @property - def is_ours(self): - """return true if this snapshot name belong to the current backup_name and snapshot formatting""" - return self.timestamp is not None From 6e89bdb7c66d6a5b8cd960b2c2affc76f495ac31 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 18 Jun 2025 23:32:28 +0200 Subject: [PATCH 063/135] fixing and refactoring --- zfs_autobackup/ZfsPointInTime.py | 30 +++++++++++++++++------------- zfs_autobackup/ZfsSnapshot.py | 2 +- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/zfs_autobackup/ZfsPointInTime.py b/zfs_autobackup/ZfsPointInTime.py index 26c3604..bcfd8aa 100644 --- a/zfs_autobackup/ZfsPointInTime.py +++ b/zfs_autobackup/ZfsPointInTime.py @@ -1,5 +1,6 @@ from zfs_autobackup.ZfsDataset import ZfsDataset from datetime import datetime +import time class ZfsPointInTime(ZfsDataset): @@ -14,8 +15,18 @@ class ZfsPointInTime(ZfsDataset): #implemented in subclass @property def prefix(self): - return "" + + + @property + def parent(self): + """get parent dataset + + :rtype: ZfsContainer | None + """ + return self.zfs_node.get_dataset(self.prefix) + + @property def timestamp(self): """get timestamp from snapshot/bookmark name. Only works for our own snapshots @@ -52,6 +63,11 @@ class ZfsPointInTime(ZfsDataset): seconds = time.mktime(dt.timetuple()) return seconds + @property + def is_ours(self): + """return true if this snapshot name belong to the current backup_name and snapshot formatting""" + return self.timestamp is not None + @property def tagless_suffix(self): @@ -71,15 +87,3 @@ class ZfsPointInTime(ZfsDataset): else: return None - @property - def is_ours(self): - """return true if this snapshot name belong to the current backup_name and snapshot formatting""" - return self.timestamp is not None - - @property - def parent(self): - """get parent dataset - - :rtype: ZfsContainer | None - """ - return self.zfs_node.get_dataset(self.prefix) diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index e6a3e47..f67563b 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -159,7 +159,7 @@ class ZfsSnapshot(ZfsPointInTime): # invalidate cache self.invalidate_cache() - # in test mode we assume everything was ok and it exists + # in test mode we pretend everything was ok and it exists if self.zfs_node.readonly: self.force_exists = True From e1d272420d8cfea89317ddfa55d86ce51da77273 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 18 Jun 2025 23:37:19 +0200 Subject: [PATCH 064/135] fixing and refactoring --- zfs_autobackup/ZfsContainer.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 6d0824b..bcc7af4 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -19,6 +19,19 @@ class ZfsContainer(ZfsDataset): self.__datasets = None # type: None|list[ZfsDataset] self.__snapshots_bookmarks = None # type: None|list[ZfsDataset] + @property + def parent(self): + """get parent dataset + + :rtype: ZfsContainer | None + """ + stripped = self.rstrip_path(1) + if stripped: + return self.zfs_node.get_dataset(stripped) + else: + return None + + def invalidate_cache(self): super().invalidate_cache() self.__written_since_ours = None @@ -286,17 +299,6 @@ class ZfsContainer(ZfsDataset): return self.__datasets - @property - def parent(self): - """get parent dataset - - :rtype: ZfsContainer | None - """ - stripped = self.rstrip_path(1) - if stripped: - return self.zfs_node.get_dataset(stripped) - else: - return None def create_filesystem(self, parents=False, unmountable=True): """create a filesystem From 91bd04d1fa8838e9880f2913ca488d96cdb9259b Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Thu, 19 Jun 2025 00:13:32 +0200 Subject: [PATCH 065/135] fixing and refactoring --- zfs_autobackup/ZfsBookmark.py | 1 - zfs_autobackup/ZfsContainer.py | 68 +++++++++++++++++----------------- zfs_autobackup/ZfsDataset.py | 3 +- zfs_autobackup/ZfsSnapshot.py | 15 +++++++- 4 files changed, 49 insertions(+), 38 deletions(-) diff --git a/zfs_autobackup/ZfsBookmark.py b/zfs_autobackup/ZfsBookmark.py index 490c244..c2d2cce 100644 --- a/zfs_autobackup/ZfsBookmark.py +++ b/zfs_autobackup/ZfsBookmark.py @@ -1,4 +1,3 @@ -from .ZfsDataset import ZfsDataset from .ExecuteNode import ExecuteError import re from datetime import datetime diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index bcc7af4..d8cd017 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -1,3 +1,4 @@ +from .ZfsBookmark import ZfsBookmark from .ZfsDataset import ZfsDataset from .ExecuteNode import ExecuteError from .ZfsSnapshot import ZfsSnapshot @@ -15,9 +16,9 @@ class ZfsContainer(ZfsDataset): super().__init__(zfs_node, name, force_exists=force_exists) self.__written_since_ours = None # type: None|int - self.__recursive_datasets = None # type: None|list[ZfsDataset] - self.__datasets = None # type: None|list[ZfsDataset] - self.__snapshots_bookmarks = None # type: None|list[ZfsDataset] + self.__recursive_datasets = None # type: None|list[ZfsContainer] + self.__datasets = None # type: None|list[ZfsContainer] + self.__snapshots_bookmarks = None # type: None|list[ZfsSnapshot|ZfsBookmark] @property def parent(self): @@ -60,7 +61,7 @@ class ZfsContainer(ZfsDataset): in raw-mode nothing is compatible. issue #219 Args: - :type target_common_snapshot: ZfsDataset + :type target_common_snapshot: ZfsSnapshot :type raw: bool """ @@ -110,35 +111,37 @@ class ZfsContainer(ZfsDataset): return True - def find_snapshot(self, snapshot): + def find_snapshot(self, snapshot_name): """find snapshot by snapshot name (can be a suffix or a different - ZfsDataset) Returns None if it cant find it. + ZfsSnapshot) Returns None if it cant find it. Note that matches with our own snapshots will be done tagless. Args: - :rtype: ZfsDataset|None - :type snapshot: str|ZfsDataset|None + :rtype: ZfsSnapshot|None + :type snapshot_name: str|ZfsSnapshot|None """ - if snapshot is None: + if snapshot_name is None: return None - if not isinstance(snapshot, ZfsDataset): - tagless_suffix = snapshot + if not isinstance(snapshot_name, ZfsDataset): + tagless_suffix = snapshot_name else: - tagless_suffix = snapshot.tagless_suffix + tagless_suffix = snapshot_name.tagless_suffix - for snapshot in self.snapshots: - if snapshot.tagless_suffix == tagless_suffix: - return snapshot + for snapshot_name in self.snapshots: + if snapshot_name.tagless_suffix == tagless_suffix: + return snapshot_name return None def find_next_snapshot(self, snapshot_bookmark): """find next snapshot in this dataset, according to snapshot or bookmark. None if it doesn't exist Args: - :type snapshot_bookmark: ZfsDataset + :type snapshot_bookmark: ZfsSnapshot|ZfsBookmark + :rtype: ZfsSnapshot|None + """ if not self.is_dataset: @@ -149,12 +152,13 @@ class ZfsContainer(ZfsDataset): if snapshot == snapshot_bookmark: found = True else: - if found and snapshot.is_snapshot: + if found and snapshot is ZfsSnapshot: return snapshot return None def mount(self, mount_point): + """Mount the dataset at mount_point, if it is a filesystem.""" self.debug("Mounting") @@ -176,11 +180,16 @@ class ZfsContainer(ZfsDataset): @property def bookmarks(self): + """get all bookmarks of this dataset + Args: + + :rtype: list[ZfsBookmark] + """ ret = [] for bookmark in self.snapshots_bookmarks: - if bookmark.is_bookmark: + if bookmark is ZfsBookmark: ret.append(bookmark) return ret @@ -232,8 +241,9 @@ class ZfsContainer(ZfsDataset): Args: :param keeps: list[snapshots] to always keep (usually the last) :param ignores: snapshots to completely ignore (usually incompatible target snapshots that are going to be destroyed anyway) - :type keeps: list[ZfsDataset] - :type ignores: list[ZfsDataset] + :type keeps: list[ZfsSnapshot] + :type ignores: list[ZfsSnapshot] + :rtype: tuple[list[ZfsSnapshot], list[ZfsSnapshot]] """ if ignores is None: @@ -266,7 +276,7 @@ class ZfsContainer(ZfsDataset): Args: :type types: str - :rtype: list[ZfsDataset] + :rtype: list[ZfsContainer] """ if self.__recursive_datasets is None: @@ -286,6 +296,8 @@ class ZfsContainer(ZfsDataset): Args: :type types: str + :rtype: list[ZfsContainer] + """ if self.__datasets is None: @@ -344,7 +356,7 @@ class ZfsContainer(ZfsDataset): def cache_snapshot_bookmark(self, snapshot, force=False): """Update our snapshot and bookmark cache (if we have any). Use force if you want to force the caching, potentially triggering a zfs list Args: - :type snapshot: ZfsDataset + :type snapshot: ZfsSnapshot|ZfsBookmark """ if force: @@ -364,18 +376,6 @@ class ZfsContainer(ZfsDataset): return ret - def find_snapshot_in_list(self, snapshots): - """return ZfsDataset from the list of snapshots, if it matches the snapshot_name. Otherwise None - Args: - :type snapshots: list[ZfsDataset] - :rtype: ZfsDataset|None - """ - - for snapshot in snapshots: - if snapshot.tagless_suffix == self.tagless_suffix: - return snapshot - - return None def find_guid_bookmark_snapshot(self, guid): """find the first bookmark or snapshot that matches, prefers bookmarks. diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 6350441..a769458 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1,5 +1,6 @@ from .ExecuteNode import ExecuteError from .ZfsContainer import ZfsContainer +from .ZfsNode import ZfsNode from .ZfsSnapshot import ZfsSnapshot @@ -28,7 +29,7 @@ class ZfsDataset: raise TypeError( "ZfsDataset should instantiated directly. Use ZfsSnapshot, ZfsBookmark or ZfsContainer instead.") - self.zfs_node = zfs_node + self.zfs_node = zfs_node # type: ZfsNode self.name = name # full actual name of dataset self.force_exists = force_exists diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index f67563b..2d5c595 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -1,4 +1,3 @@ -from .ZfsDataset import ZfsDataset from .ExecuteNode import ExecuteError import re import sys @@ -33,6 +32,18 @@ class ZfsSnapshot(ZfsPointInTime): if pattern.search(suffix) is not None: self.debug("Excluded (path matches snapshot exclude pattern)") return True + # def find_snapshot_in_list(self, snapshots): + # """return ZfsDataset from the list of snapshots, if it matches the snapshot_name. Otherwise None + # Args: + # :type snapshots: list[ZfsSnapshot|ZfsBookmark] + # :rtype: ZfsDataset|None + # """ + # + # for snapshot in snapshots: + # if snapshot.tagless_suffix == self.tagless_suffix: + # return snapshot + # + # return None def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, write_embedded, send_pipes, zfs_compressed): @@ -249,7 +260,7 @@ class ZfsSnapshot(ZfsPointInTime): self.zfs_node.run(cmd=cmd) bookmark = self.zfs_node.get_dataset(bookmark_name, force_exists=True) - self.cache_snapshot_bookmark(bookmark) + self.parent.cache_snapshot_bookmark(bookmark) return bookmark @property From 101125b9dc8bc1378a55ce68bb5ee1d2d99c4f9f Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Thu, 19 Jun 2025 00:18:37 +0200 Subject: [PATCH 066/135] fixing and refactoring --- zfs_autobackup/ZfsContainer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index d8cd017..5e5dcb2 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -381,7 +381,7 @@ class ZfsContainer(ZfsDataset): """find the first bookmark or snapshot that matches, prefers bookmarks. Args: :type guid:str - :rtype: ZfsDataset|None + :rtype: ZfsBookmark|ZfsSnapshot|None """ # Since this is slower, we only use it if the name matching with find_snapshot and find_bookmark doesn work. @@ -400,12 +400,12 @@ class ZfsContainer(ZfsDataset): ZfsDataset) Returns None if it cant find it. We try to find the bookmark with the preferred tag (which is usually a target path guid, to prevent conflicting bookmarks by multiple sends) - After that we return any bookmark that matches (and ignore the tag) + If that fails, we return any bookmark that matches (and ignore the tag) Args: - :rtype: ZfsDataset|None - :type snapshot_bookmark: str|ZfsDataset|None - :type ignore_tag: bool + :rtype: ZfsBookmark|None + :type snapshot_bookmark: str|ZfsBookmark|ZfsSnapshot|None + :type preferred_tag: str """ if snapshot_bookmark is None: From fac17d40122ce86df177744287d34b98d7edf638 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Thu, 19 Jun 2025 00:22:17 +0200 Subject: [PATCH 067/135] fixing and refactoring --- zfs_autobackup/ZfsContainer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 5e5dcb2..d5ff4ca 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -429,7 +429,7 @@ class ZfsContainer(ZfsDataset): def find_exact_bookmark(self, bookmark_name): """find exact bookmark name, or retruns none - :rtype: ZfsDataset|None + :rtype: ZfsBookmark|None """ for snapshot_bookmark in self.bookmarks: @@ -442,7 +442,8 @@ class ZfsContainer(ZfsDataset): ZfsDataset) Args: - :type snapshot: str or ZfsDataset + :type snapshot: str or ZfsSnapshot + :rtype: int|None """ if not isinstance(snapshot, ZfsDataset): From 2217e2ad0107db3d2f550138e264abd1df48415e Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Fri, 20 Jun 2025 18:47:51 +0200 Subject: [PATCH 068/135] fixing and refactoring --- zfs_autobackup/ZfsContainer.py | 96 ++++++++++++++++---------------- zfs_autobackup/ZfsPointInTime.py | 2 + zfs_autobackup/ZfsSnapshot.py | 27 +++++---- 3 files changed, 64 insertions(+), 61 deletions(-) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index d5ff4ca..82d44df 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -53,6 +53,42 @@ class ZfsContainer(ZfsDataset): return ret + @property + def bookmarks(self): + """get all bookmarks of this dataset + Args: + + :rtype: list[ZfsBookmark] + """ + + ret = [] + + for bookmark in self.snapshots_bookmarks: + if bookmark is ZfsBookmark: + ret.append(bookmark) + + return ret + + @property + def snapshots_bookmarks(self): + """get all snapshots and bookmarks of this dataset (ordered by createtxg, so its suitable to determine incremental zfs send order) + :rtype: list[ZfsSnapshot|ZfsBookmark] + """ + + # cached? + if self.__snapshots_bookmarks is None: + self.debug("Getting snapshots and bookmarks") + + cmd = [ + "zfs", "list", "-d", "1", "-r", "-t", "snapshot,bookmark", "-H", "-o", "name", "-s", "createtxg", + self.name + ] + + self.__snapshots_bookmarks = self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), + force_exists=True) + + return self.__snapshots_bookmarks + def find_incompatible_snapshots(self, target_common_snapshot, raw): """returns a list[snapshots] that is incompatible for a zfs recv onto the common_snapshot. all direct followup snapshots with written=0 are @@ -144,8 +180,7 @@ class ZfsContainer(ZfsDataset): """ - if not self.is_dataset: - raise (Exception("Please call this on a dataset.")) + found = False for snapshot in self.snapshots_bookmarks: @@ -178,21 +213,6 @@ class ZfsContainer(ZfsDataset): self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - @property - def bookmarks(self): - """get all bookmarks of this dataset - Args: - - :rtype: list[ZfsBookmark] - """ - - ret = [] - - for bookmark in self.snapshots_bookmarks: - if bookmark is ZfsBookmark: - ret.append(bookmark) - - return ret def automount(self): """Mount the dataset as if one did a zfs mount -a, but only for this dataset @@ -313,7 +333,7 @@ class ZfsContainer(ZfsDataset): def create_filesystem(self, parents=False, unmountable=True): - """create a filesystem + """create this container as a filesystem Args: :type parents: bool @@ -333,25 +353,6 @@ class ZfsContainer(ZfsDataset): self.force_exists = True - @property - def snapshots_bookmarks(self): - """get all snapshots and bookmarks of this dataset (ordered by createtxg, so its suitable to determine incremental zfs send order) - :rtype: list[ZfsSnapshot|ZfsBookmark] - """ - - # cached? - if self.__snapshots_bookmarks is None: - self.debug("Getting snapshots and bookmarks") - - cmd = [ - "zfs", "list", "-d", "1", "-r", "-t", "snapshot,bookmark", "-H", "-o", "name", "-s", "createtxg", - self.name - ] - - self.__snapshots_bookmarks = self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), - force_exists=True) - - return self.__snapshots_bookmarks def cache_snapshot_bookmark(self, snapshot, force=False): """Update our snapshot and bookmark cache (if we have any). Use force if you want to force the caching, potentially triggering a zfs list @@ -466,9 +467,10 @@ class ZfsContainer(ZfsDataset): On the source it prefers the specified bookmark_name Args: - :rtype: tuple[ZfsDataset, ZfsDataset] | tuple[None,None] + :rtype: tuple[ZfsSnapshot|ZfsBookmark, ZfsSnapshot] | tuple[None,None] + :returns: (source_common_snapshot, target_common_snapshot) :type guid_check: bool - :type target_dataset: ZfsDataset + :type target_dataset: ZfsContainer :type bookmark_tag: str """ @@ -509,21 +511,21 @@ class ZfsContainer(ZfsDataset): raise (Exception("Cant find common bookmark or snapshot with target.")) def _pre_clean(self, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): - """cleanup old stuff before starting snapshot syncing + """cleanup old stuff on the source before starting snapshot syncing Args: - :type source_common_snapshot: ZfsDataset - :type target_dataset: ZfsDataset - :type source_obsoletes: list[ZfsDataset] - :type target_obsoletes: list[ZfsDataset] - :type target_transfers: list[ZfsDataset] + :type source_common_snapshot: ZfsSnapshot|ZfsBookmark|None + :type target_dataset: ZfsContainer + :type source_obsoletes: list[ZfsSnapshot] + :type target_obsoletes: list[ZfsSnapshot] + :type target_transfers: list[ZfsSnapshot] """ # on source: delete all obsoletes that are not in target_transfers (except common snapshot, if its not a bookmark) for source_snapshot in self.snapshots: if (source_snapshot in source_obsoletes and source_common_snapshot != source_snapshot - and source_snapshot.find_snapshot_in_list(target_transfers) is None): + and source_snapshot.find_snapshot_by_suffix(target_transfers) is None): source_snapshot.destroy() # on target: destroy everything thats obsolete, except common_snapshot @@ -606,7 +608,7 @@ class ZfsContainer(ZfsDataset): self.verbose("Selected") return True - def destroy(self, fail_exception=False): + def destroy(self, fail_exception=False, **kwargs): self.verbose("Destroying") return super().destroy(fail_exception=fail_exception) diff --git a/zfs_autobackup/ZfsPointInTime.py b/zfs_autobackup/ZfsPointInTime.py index bcfd8aa..9ff919c 100644 --- a/zfs_autobackup/ZfsPointInTime.py +++ b/zfs_autobackup/ZfsPointInTime.py @@ -1,3 +1,5 @@ +import sys + from zfs_autobackup.ZfsDataset import ZfsDataset from datetime import datetime import time diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index 2d5c595..97cdbb0 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -32,18 +32,19 @@ class ZfsSnapshot(ZfsPointInTime): if pattern.search(suffix) is not None: self.debug("Excluded (path matches snapshot exclude pattern)") return True - # def find_snapshot_in_list(self, snapshots): - # """return ZfsDataset from the list of snapshots, if it matches the snapshot_name. Otherwise None - # Args: - # :type snapshots: list[ZfsSnapshot|ZfsBookmark] - # :rtype: ZfsDataset|None - # """ - # - # for snapshot in snapshots: - # if snapshot.tagless_suffix == self.tagless_suffix: - # return snapshot - # - # return None + + def find_snapshot_by_suffix(self, snapshot_bookmarks): + """return ZfsSnapshot|ZfsBookmark from the list of snapshot_bookmarks, if it matches our snapshot_name. Otherwise None + Args: + :type snapshot_bookmarks: list[ZfsSnapshot|ZfsBookmark] + :rtype: ZfsSnapshot|ZfsBookmark|None + """ + + for snapshot_bookmark in snapshot_bookmarks: + if snapshot_bookmark.tagless_suffix == self.tagless_suffix: + return snapshot_bookmark + + return None def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, write_embedded, send_pipes, zfs_compressed): @@ -241,8 +242,6 @@ class ZfsSnapshot(ZfsPointInTime): # NOTE: we use the tag to add the target_path GUID, so that we can have multiple bookmarks for the same snapshot, but for different targets. # This is to make sure you can send a backup to two locations, without them interfering with eachothers bookmarks. - if not self.is_snapshot: - raise (Exception("Can only bookmark a snapshot!")) bookmark_name = self.prefix + "#" + self.tagless_suffix + self.zfs_node.tag_seperator + tag From 8b64bac6740fb7ac3037bbcba7e09c283731f2fe Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Mon, 23 Jun 2025 23:21:24 +0200 Subject: [PATCH 069/135] fixing and refactoring --- tests/test_zfsautobackup40.py | 3 ++- zfs_autobackup/ZfsContainer.py | 38 ++++++++++++++++---------------- zfs_autobackup/ZfsDataset.py | 9 ++++---- zfs_autobackup/ZfsNode.py | 8 +++---- zfs_autobackup/ZfsPointInTime.py | 12 +++++++++- 5 files changed, 40 insertions(+), 30 deletions(-) diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py index 98fd763..07d07c6 100644 --- a/tests/test_zfsautobackup40.py +++ b/tests/test_zfsautobackup40.py @@ -560,4 +560,5 @@ test_target1/test_source1/fs1/sub@test2-20010203000000 # TODO # - check if holds and no-holds function ok when bookmarks are enabled -# -delete bookmarks we have on other snapshots +# - delete bookmarks we have on other snapshots +# - test if resuming works with bookmarks diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 82d44df..b93d546 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -2,10 +2,10 @@ from .ZfsBookmark import ZfsBookmark from .ZfsDataset import ZfsDataset from .ExecuteNode import ExecuteError from .ZfsSnapshot import ZfsSnapshot -import re -from datetime import datetime -import sys -import time +# import re +# from datetime import datetime +# import sys +# import time class ZfsContainer(ZfsDataset): @@ -13,6 +13,7 @@ class ZfsContainer(ZfsDataset): def __init__(self, zfs_node, name, force_exists=None): + super().__init__(zfs_node, name, force_exists=force_exists) self.__written_since_ours = None # type: None|int @@ -623,7 +624,7 @@ class ZfsContainer(ZfsDataset): Args: :type send_pipes: list[str] :type recv_pipes: list[str] - :type target_dataset: ZfsDataset + :type target_dataset: ZfsContainer :type features: list[str] :type show_progress: bool :type filter_properties: list[str] @@ -765,7 +766,7 @@ class ZfsContainer(ZfsDataset): what to do Args: - :type incompatible_target_snapshots: list[ZfsDataset] + :type incompatible_target_snapshots: list[ZfsSnapshot] :type destroy_incompatible: bool """ @@ -788,8 +789,8 @@ class ZfsContainer(ZfsDataset): """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. Args: - :rtype: ( ZfsDataset, list[ZfsDataset], list[ZfsDataset], list[ZfsDataset], list[ZfsDataset] ) - :type target_dataset: ZfsDataset + :rtype: ( ZfsSnapshot|ZfsBookmark, list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot] ) + :type target_dataset: ZfsContainer :type also_other_snapshots: bool :type guid_check: bool :type raw: bool @@ -797,13 +798,11 @@ class ZfsContainer(ZfsDataset): Returns: tuple: A tuple containing: - - ZfsDataset: The common snapshot - - list[ZfsDataset]: Our obsolete source snapshots, after transfer is done. (will be thinned asap) - - list[ZfsDataset]: Our obsolete target snapshots, after transfer is done. (will be thinned asap) - - list[ZfsDataset]: Transfer target snapshots. These need to be transferred. - - list[ZfsDataset]: Incompatible target snapshots. Target snapshots that are in the way, after the common snapshot. (need to be destroyed to continue) - - + - ZfsSnapshot|ZfsBookmark: The source common snapshot + - list[ZfsSnapshot]: Our obsolete source snapshots, after transfer is done. (will be thinned asap) + - list[ZfsSnapshot]: Our obsolete target snapshots, after transfer is done. (will be thinned asap) + - list[ZfsSnapshot]: Transfer target snapshots. These need to be transferred. + - list[ZfsSnapshot]: Incompatible target snapshots. Target snapshots that are in the way, after the common snapshot. (need to be destroyed to continue) """ ### 1: determine common and start snapshot @@ -844,7 +843,7 @@ class ZfsContainer(ZfsDataset): if (also_other_snapshots or source_snapshot.is_ours) and not source_snapshot.is_snapshot_excluded: # create virtual target snapshot target_snapshot = target_dataset.zfs_node.get_dataset( - target_dataset.prefix + source_snapshot.typed_suffix, force_exists=False) + target_dataset.name + source_snapshot.typed_suffix, force_exists=False) possible_target_snapshots.append(target_snapshot) source_snapshot = self.find_next_snapshot(source_snapshot) @@ -870,8 +869,9 @@ class ZfsContainer(ZfsDataset): """validate and get (or destory) resume token Args: - :type target_dataset: ZfsDataset - :type start_snapshot: ZfsDataset + :type target_dataset: ZfsContainer + :type start_snapshot: ZfsSnapshot|None + :rtype: str|None """ if target_dataset.exists and 'receive_resume_token' in target_dataset.properties: @@ -881,7 +881,7 @@ class ZfsContainer(ZfsDataset): else: resume_token = target_dataset.properties['receive_resume_token'] # not valid anymore? - resume_snapshot = self.get_resume_snapshot(resume_token) + resume_snapshot = self.zfs_node.get_resume_snapshot(resume_token) if not resume_snapshot or start_snapshot.suffix != resume_snapshot.suffix: target_dataset.verbose("Aborting resume, its no longer valid.") target_dataset.abort_resume() diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index a769458..f7aa9f9 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1,7 +1,4 @@ from .ExecuteNode import ExecuteError -from .ZfsContainer import ZfsContainer -from .ZfsNode import ZfsNode -from .ZfsSnapshot import ZfsSnapshot class ZfsDataset: @@ -25,9 +22,11 @@ class ZfsDataset: """ # prevent direct instantiation - if type(self) is not ZfsContainer and not ZfsSnapshot and not ZfsDataset: + if type(self) is ZfsDataset: raise TypeError( - "ZfsDataset should instantiated directly. Use ZfsSnapshot, ZfsBookmark or ZfsContainer instead.") + "should not be instantiated directly.") + + from .ZfsNode import ZfsNode self.zfs_node = zfs_node # type: ZfsNode self.name = name # full actual name of dataset diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index cbdd5f7..d84dfcb 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -334,17 +334,17 @@ class ZfsNode(ExecuteNode): return (selected_filesystems, excluded_filesystems) - def get_resume_snapshot(self, resume_token): + def get_resume_snapshot(self, target_resume_token): """returns snapshot that will be resumed by this resume token (run this on source node with target-token) Args: - :type resume_token: str - :rtype: ZfsDataset|None + :type target_resume_token: str + :rtype: ZfsSnapshot|None """ # use zfs send -n option to determine this # NOTE: on smartos stderr, on linux stdout - (stdout, stderr) = self.run(["zfs", "send", "-t", resume_token, "-n", "-v"], valid_exitcodes=[0, 255], + (stdout, stderr) = self.run(["zfs", "send", "-t", target_resume_token, "-n", "-v"], valid_exitcodes=[0, 255], readonly=True, return_stderr=True) if stdout: lines = stdout diff --git a/zfs_autobackup/ZfsPointInTime.py b/zfs_autobackup/ZfsPointInTime.py index 9ff919c..247a411 100644 --- a/zfs_autobackup/ZfsPointInTime.py +++ b/zfs_autobackup/ZfsPointInTime.py @@ -1,6 +1,6 @@ import sys -from zfs_autobackup.ZfsDataset import ZfsDataset +from .ZfsDataset import ZfsDataset from datetime import datetime import time @@ -9,6 +9,16 @@ class ZfsPointInTime(ZfsDataset): """contains stuff that is common to snapshots and bookmarks""" + def __init__(self, zfs_node, name, force_exists=None): + + + if type(self) is ZfsPointInTime: + raise TypeError( + "should not be instantiated directly.") + + super().__init__(zfs_node, name, force_exists) + + #implemented in subclass @property def suffix(self): From a0a2658b9542ccfcba6e2c2869ce7e59cdf49166 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Jun 2025 00:37:18 +0200 Subject: [PATCH 070/135] fixing and refactoring --- tests/test_zfsnode.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_zfsnode.py b/tests/test_zfsnode.py index f1f2d70..67fc1a8 100644 --- a/tests/test_zfsnode.py +++ b/tests/test_zfsnode.py @@ -1,6 +1,7 @@ from basetest import * from zfs_autobackup.LogStub import LogStub from zfs_autobackup.ExecuteNode import ExecuteError +from zfs_autobackup.ZfsSnapshot import ZfsSnapshot class TestZfsNode(unittest2.TestCase): @@ -162,9 +163,9 @@ test_target1 for node in [node_local, node_utc]: with self.subTest("timestamp ordering " + ("utc" if node == node_utc else "localtime")): - dataset_a = ZfsDataset(node, "test_source1@test-20101111000001") - dataset_b = ZfsDataset(node, "test_source1@test-20101111000002") - dataset_c = ZfsDataset(node, "test_source1@test-20240101020202") + dataset_a = ZfsSnapshot(node, "test_source1@test-20101111000001") + dataset_b = ZfsSnapshot(node, "test_source1@test-20101111000002") + dataset_c = ZfsSnapshot(node, "test_source1@test-20240101020202") self.assertGreater(dataset_b.timestamp, dataset_a.timestamp) self.assertGreater(dataset_c.timestamp, dataset_b.timestamp) From 83b9b6d50d5949cf7dfc978d41d909c8527e1573 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Jun 2025 08:37:48 +0200 Subject: [PATCH 071/135] fixing and refactoring --- zfs_autobackup/ZfsContainer.py | 17 +++++++++++++++++ zfs_autobackup/ZfsDataset.py | 15 --------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index b93d546..e036d09 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -129,6 +129,22 @@ class ZfsContainer(ZfsDataset): return self.__written_since_ours + def is_changed(self, min_changed_bytes=1): + """dataset is changed since ANY latest snapshot ? + + Args: + :type min_changed_bytes: int + """ + self.debug("Checking if dataset is changed") + + if min_changed_bytes == 0: + return True + + if int(self.properties['written']) < min_changed_bytes: + return False + else: + return True + def is_changed_ours(self, min_changed_bytes=1): """dataset is changed since OUR latest snapshot? @@ -146,6 +162,7 @@ class ZfsContainer(ZfsDataset): if self.written_since_ours < min_changed_bytes: return False + return True def find_snapshot(self, snapshot_name): diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index f7aa9f9..b5b1f4b 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -180,21 +180,6 @@ class ZfsDataset: return self.__properties - def is_changed(self, min_changed_bytes=1): - """dataset is changed since ANY latest snapshot ? - - Args: - :type min_changed_bytes: int - """ - self.debug("Checking if dataset is changed") - - if min_changed_bytes == 0: - return True - - if int(self.properties['written']) < min_changed_bytes: - return False - else: - return True def get_allowed_properties(self, filter_properties, set_properties): """only returns lists of allowed properties for this dataset type From 3222f14aa1244be724592fce91933dbed6664e30 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Jun 2025 23:14:55 +0200 Subject: [PATCH 072/135] fixing and refactoring --- zfs_autobackup/ZfsContainer.py | 14 ++++++++------ zfs_autobackup/ZfsNode.py | 4 ++-- zfs_autobackup/ZfsPointInTime.py | 10 +++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index e036d09..92d78b4 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -49,9 +49,11 @@ class ZfsContainer(ZfsDataset): ret = [] for snapshot in self.snapshots_bookmarks: - if snapshot is ZfsSnapshot: + #check is snapshot is a ZfsSnapshot, and append to ret if it is + if isinstance(snapshot, ZfsSnapshot): ret.append(snapshot) + return ret @property @@ -65,7 +67,7 @@ class ZfsContainer(ZfsDataset): ret = [] for bookmark in self.snapshots_bookmarks: - if bookmark is ZfsBookmark: + if type(bookmark) is ZfsBookmark: ret.append(bookmark) return ret @@ -135,7 +137,7 @@ class ZfsContainer(ZfsDataset): Args: :type min_changed_bytes: int """ - self.debug("Checking if dataset is changed") + self.debug("Checking if dataset is changed {} or more bytes".format(min_changed_bytes)) if min_changed_bytes == 0: return True @@ -205,7 +207,7 @@ class ZfsContainer(ZfsDataset): if snapshot == snapshot_bookmark: found = True else: - if found and snapshot is ZfsSnapshot: + if found and type(snapshot) is ZfsSnapshot: return snapshot return None @@ -748,7 +750,7 @@ class ZfsContainer(ZfsDataset): # TODO: make a better is_ours specially for bookmarks, as part of the next refactoring splitting in more classes # delete any bookmark that ends in ours tag_seprator + tag. - if prev_source_snapshot_bookmark and prev_source_snapshot_bookmark.is_bookmark and prev_source_snapshot_bookmark.name.endswith( + if prev_source_snapshot_bookmark and type(prev_source_snapshot_bookmark) is ZfsBookmark and prev_source_snapshot_bookmark.name.endswith( self.zfs_node.tag_seperator + bookmark_tag): prev_source_snapshot_bookmark.destroy() @@ -760,7 +762,7 @@ class ZfsContainer(ZfsDataset): source_snapshot.hold() # release hold, cleanup obsolete snapshot - if prev_source_snapshot_bookmark and prev_source_snapshot_bookmark.is_snapshot: + if prev_source_snapshot_bookmark and type(prev_source_snapshot_bookmark) is ZfsSnapshot: prev_source_snapshot_bookmark.release() if prev_source_snapshot_bookmark in source_obsoletes: prev_source_snapshot_bookmark.destroy() diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index d84dfcb..2d451c8 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -238,7 +238,7 @@ class ZfsNode(ExecuteNode): :type datasets: list[ZfsContainer] """ - pools = {} + pools = {} # type: dict[str, list[ZfsSnapshot]] # collect snapshots that we want to make, per pool # self.debug(datasets) @@ -273,7 +273,7 @@ class ZfsNode(ExecuteNode): for snapshot_property in set_snapshot_properties: cmd += ['-o', snapshot_property] - cmd.extend(map(lambda snapshot_: str(snapshot_), snapshots)) + cmd.extend(map(lambda snapshot_: snapshot_.name, snapshots)) self.verbose("Creating snapshots {} in pool {}".format(snapshot_name, pool_name)) self.run(cmd, readonly=False) diff --git a/zfs_autobackup/ZfsPointInTime.py b/zfs_autobackup/ZfsPointInTime.py index 247a411..f0144a9 100644 --- a/zfs_autobackup/ZfsPointInTime.py +++ b/zfs_autobackup/ZfsPointInTime.py @@ -49,13 +49,13 @@ class ZfsPointInTime(ZfsDataset): :rtype: int|None """ - if self.zfs_node.tag_seperator in self.suffix: - tagless_suffix = self.suffix.split(self.zfs_node.tag_seperator)[0] - else: - tagless_suffix = self.suffix + # if self.zfs_node.tag_seperator in self.suffix: + # tagless_suffix = self.suffix.split(self.zfs_node.tag_seperator)[0] + # else: + # tagless_suffix = self.suffix try: - dt = datetime.strptime(tagless_suffix, self.zfs_node.snapshot_time_format) + dt = datetime.strptime(self.tagless_suffix, self.zfs_node.snapshot_time_format) except ValueError: return None From c3102e0236e4c58bf318886c52403499bf0af16a Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 24 Jun 2025 23:16:15 +0200 Subject: [PATCH 073/135] fixing and refactoring --- zfs_autobackup/ZfsPointInTime.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/zfs_autobackup/ZfsPointInTime.py b/zfs_autobackup/ZfsPointInTime.py index f0144a9..b30af10 100644 --- a/zfs_autobackup/ZfsPointInTime.py +++ b/zfs_autobackup/ZfsPointInTime.py @@ -49,13 +49,14 @@ class ZfsPointInTime(ZfsDataset): :rtype: int|None """ - # if self.zfs_node.tag_seperator in self.suffix: - # tagless_suffix = self.suffix.split(self.zfs_node.tag_seperator)[0] - # else: - # tagless_suffix = self.suffix + #note: cant use self.tagless_suffix here, because it depends on us + if self.zfs_node.tag_seperator in self.suffix: + tagless_suffix = self.suffix.split(self.zfs_node.tag_seperator)[0] + else: + tagless_suffix = self.suffix try: - dt = datetime.strptime(self.tagless_suffix, self.zfs_node.snapshot_time_format) + dt = datetime.strptime(tagless_suffix, self.zfs_node.snapshot_time_format) except ValueError: return None From 33aa6cdb7f5240218ab57f7ad92cad08f2341c0b Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 25 Jun 2025 00:10:28 +0200 Subject: [PATCH 074/135] fixing and refactoring. almost there --- zfs_autobackup/ExecuteNode.py | 3 +++ zfs_autobackup/ZfsDataset.py | 1 - zfs_autobackup/ZfsSnapshot.py | 13 ++++++++----- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/zfs_autobackup/ExecuteNode.py b/zfs_autobackup/ExecuteNode.py index 1200eed..892729d 100644 --- a/zfs_autobackup/ExecuteNode.py +++ b/zfs_autobackup/ExecuteNode.py @@ -100,8 +100,10 @@ class ExecuteNode(LogStub): :param cmd: the actual command, should be a list, where the first item is the command and the rest are parameters. use ExecuteNode.PIPE to add an unescaped | (if you want to use system piping instead of python piping) + :type cmd: list[str|ExecuteNode.PIPE] :param pipe: return CmdPipe instead of executing it. (pipe this into another run() command via inp=...) :param inp: Can be None, a string or a CmdPipe that was previously returned. + :type inp: CmdPipe | str | None :param tab_split: split tabbed files in output into a list :param valid_exitcodes: list of valid exit codes for this command. Use [] to accept all exit codes. Default [0] :param readonly: make this True if the command doesn't make any changes and is safe to execute in testmode @@ -109,6 +111,7 @@ class ExecuteNode(LogStub): :param return_stderr: return both stdout and stderr as a tuple. (normally only returns stdout) :param return_all: return both stdout and stderr and exit_code as a tuple. (normally only returns stdout) :param cwd: Change current working directory before executing command. + :rtype: list[str] | tuple[list[str], list[str]] | tuple[list[str], list[str], int] | CmdPipe """ diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index b5b1f4b..5ea1dbb 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1,6 +1,5 @@ from .ExecuteNode import ExecuteError - class ZfsDataset: """A generic ZFS dataset, this has all the common functions of zfs filesystems, volumes, snapshots and bookmarks. diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index 97cdbb0..0b8242f 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -26,7 +26,7 @@ class ZfsSnapshot(ZfsPointInTime): @property def is_snapshot_excluded(self): - """true if this dataset is a snapshot and matches the exclude pattern""" + """true if this is a snapshot and matches the exclude pattern""" suffix = self.suffix for pattern in self.zfs_node.exclude_snapshot_patterns: if pattern.search(suffix) is not None: @@ -57,7 +57,7 @@ class ZfsSnapshot(ZfsPointInTime): :param send_pipes: output cmd array that will be added to actual zfs send command. (e.g. mbuffer or compression program) :type send_pipes: list[str] :type features: list[str] - :type prev_snapshot: ZfsDataset + :type prev_snapshot: ZfsSnapshot|ZfsBookmark|None :type resume_token: str :type show_progress: bool :type raw: bool @@ -274,12 +274,12 @@ class ZfsSnapshot(ZfsPointInTime): return self.zfs_node.hold_name in self.holds def hold(self): - """hold dataset""" + """hold snapshot""" self.debug("holding") self.zfs_node.run(["zfs", "hold", self.zfs_node.hold_name, self.name], valid_exitcodes=[0, 1]) def release(self): - """release dataset""" + """release snapshot""" if self.zfs_node.readonly or self.is_hold(): self.debug("releasing") self.zfs_node.run(["zfs", "release", self.zfs_node.hold_name, self.name], valid_exitcodes=[0, 1]) @@ -293,10 +293,13 @@ class ZfsSnapshot(ZfsPointInTime): def destroy(self, fail_exception=False, defered=False): self.verbose("Destroying") + self.release() return super().destroy(fail_exception=fail_exception, deferred=defered) def clone(self, name): - """clones this snapshot and returns ZfsDataset of the clone""" + """clones this snapshot and returns ZfsContainer of the clone + :rtype: ZfsContainer + """ self.debug("Cloning to {}".format(name)) From 9e3d4c0a7452923c99390bef3020ffcc27d9eeec Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 25 Jun 2025 00:13:16 +0200 Subject: [PATCH 075/135] fixing and refactoring. almost there --- tests/test_zfsautobackup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 7115d7c..26b34a8 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -4,6 +4,7 @@ from basetest import * import time from zfs_autobackup.LogConsole import LogConsole +from zfs_autobackup.ZfsSnapshot import ZfsSnapshot class TestZfsAutobackup(unittest2.TestCase): @@ -898,7 +899,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 n = ZfsNode(utc=False, snapshot_time_format="bla", hold_name="bla", logger=l, readonly=False, debug_output=False, thinner=None, exclude_snapshot_patterns=None, ssh_to=None, ssh_config=None, description=None, tag_seperator="_") - d = ZfsDataset(n, "test_source1@test") + d = ZfsSnapshot(n, "test_source1@test") sp = d.send_pipe([], prev_snapshot=None, resume_token=None, show_progress=True, raw=False, send_pipes=[], send_properties=True, write_embedded=True, zfs_compressed=True) From 4453959dce3ac34d27bd516fec2b01a55d27a599 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 25 Jun 2025 10:37:43 +0200 Subject: [PATCH 076/135] refactor fixes --- zfs_autobackup/ZfsCheck.py | 20 +++++++++++++++--- zfs_autobackup/ZfsContainer.py | 38 +++++----------------------------- zfs_autobackup/ZfsDataset.py | 27 ++++++++++++++++++++---- zfs_autobackup/util.py | 3 +++ 4 files changed, 48 insertions(+), 40 deletions(-) diff --git a/zfs_autobackup/ZfsCheck.py b/zfs_autobackup/ZfsCheck.py index 0f410f8..68048d5 100644 --- a/zfs_autobackup/ZfsCheck.py +++ b/zfs_autobackup/ZfsCheck.py @@ -9,8 +9,10 @@ from .BlockHasher import BlockHasher from .ZfsNode import ZfsNode from .util import * from .CliBase import CliBase +from .ZfsSnapshot import ZfsSnapshot +# NOTE: zfscheck has no ssh support, because it would be too problematic during cleanup of temporary mounts and clones. class ZfsCheck(CliBase): def __init__(self, argv, print_arguments=True): @@ -68,7 +70,10 @@ class ZfsCheck(CliBase): return args def prepare_zfs_filesystem(self, snapshot): - + """ + :type snapshot: ZfsSnapshot + :rtype: str + """ mnt = "/tmp/" + tmp_name() self.debug("Create temporary mount point {}".format(mnt)) self.node.run(["mkdir", mnt]) @@ -76,6 +81,9 @@ class ZfsCheck(CliBase): return mnt def cleanup_zfs_filesystem(self, snapshot): + """ + :type snapshot: ZfsSnapshot + """ mnt = "/tmp/" + tmp_name() snapshot.unmount(mnt) self.debug("Cleaning up temporary mount point") @@ -84,7 +92,10 @@ class ZfsCheck(CliBase): # NOTE: https://www.google.com/search?q=Mount+Path+Limit+freebsd # Freebsd has limitations regarding path length, so we have to clone it so the part stays sort def prepare_zfs_volume(self, snapshot): - """clone volume, waits and tries to findout /dev path to the volume, in a compatible way. (linux/freebsd/smartos)""" + """clone volume, waits and tries to findout /dev path to the volume, in a compatible way. (linux/freebsd/smartos) + :type snapshot: ZfsSnapshot + :rtype: str + """ clone_name = get_tmp_clone_name(snapshot) clone = snapshot.clone(clone_name) @@ -114,7 +125,10 @@ class ZfsCheck(CliBase): locations))) def cleanup_zfs_volume(self, snapshot): - """destroys temporary volume snapshot""" + """destroys temporary volume snapshot + :type snapshot: ZfsSnapshot + + """ clone_name = get_tmp_clone_name(snapshot) clone = snapshot.zfs_node.get_dataset(clone_name) clone.destroy() diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 92d78b4..d9f2df7 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -2,6 +2,8 @@ from .ZfsBookmark import ZfsBookmark from .ZfsDataset import ZfsDataset from .ExecuteNode import ExecuteError from .ZfsSnapshot import ZfsSnapshot + + # import re # from datetime import datetime # import sys @@ -13,7 +15,6 @@ class ZfsContainer(ZfsDataset): def __init__(self, zfs_node, name, force_exists=None): - super().__init__(zfs_node, name, force_exists=force_exists) self.__written_since_ours = None # type: None|int @@ -33,7 +34,6 @@ class ZfsContainer(ZfsDataset): else: return None - def invalidate_cache(self): super().invalidate_cache() self.__written_since_ours = None @@ -49,11 +49,10 @@ class ZfsContainer(ZfsDataset): ret = [] for snapshot in self.snapshots_bookmarks: - #check is snapshot is a ZfsSnapshot, and append to ret if it is + # check is snapshot is a ZfsSnapshot, and append to ret if it is if isinstance(snapshot, ZfsSnapshot): ret.append(snapshot) - return ret @property @@ -164,7 +163,6 @@ class ZfsContainer(ZfsDataset): if self.written_since_ours < min_changed_bytes: return False - return True def find_snapshot(self, snapshot_name): @@ -200,8 +198,6 @@ class ZfsContainer(ZfsDataset): """ - - found = False for snapshot in self.snapshots_bookmarks: if snapshot == snapshot_bookmark: @@ -212,28 +208,6 @@ class ZfsContainer(ZfsDataset): return None - def mount(self, mount_point): - """Mount the dataset at mount_point, if it is a filesystem.""" - - self.debug("Mounting") - - cmd = [ - "mount", "-tzfs", self.name, mount_point - ] - - self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - - def unmount(self, mount_point): - - self.debug("Unmounting") - - cmd = [ - "umount", mount_point - ] - - self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - - def automount(self): """Mount the dataset as if one did a zfs mount -a, but only for this dataset Failure to mount doesnt result in an exception, but outputs errors to STDERR. @@ -351,7 +325,6 @@ class ZfsContainer(ZfsDataset): return self.__datasets - def create_filesystem(self, parents=False, unmountable=True): """create this container as a filesystem @@ -373,7 +346,6 @@ class ZfsContainer(ZfsDataset): self.force_exists = True - def cache_snapshot_bookmark(self, snapshot, force=False): """Update our snapshot and bookmark cache (if we have any). Use force if you want to force the caching, potentially triggering a zfs list Args: @@ -397,7 +369,6 @@ class ZfsContainer(ZfsDataset): return ret - def find_guid_bookmark_snapshot(self, guid): """find the first bookmark or snapshot that matches, prefers bookmarks. Args: @@ -750,7 +721,8 @@ class ZfsContainer(ZfsDataset): # TODO: make a better is_ours specially for bookmarks, as part of the next refactoring splitting in more classes # delete any bookmark that ends in ours tag_seprator + tag. - if prev_source_snapshot_bookmark and type(prev_source_snapshot_bookmark) is ZfsBookmark and prev_source_snapshot_bookmark.name.endswith( + if prev_source_snapshot_bookmark and type( + prev_source_snapshot_bookmark) is ZfsBookmark and prev_source_snapshot_bookmark.name.endswith( self.zfs_node.tag_seperator + bookmark_tag): prev_source_snapshot_bookmark.destroy() diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 5ea1dbb..9d04398 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1,5 +1,6 @@ from .ExecuteNode import ExecuteError + class ZfsDataset: """A generic ZFS dataset, this has all the common functions of zfs filesystems, volumes, snapshots and bookmarks. @@ -25,9 +26,7 @@ class ZfsDataset: raise TypeError( "should not be instantiated directly.") - from .ZfsNode import ZfsNode - - self.zfs_node = zfs_node # type: ZfsNode + self.zfs_node = zfs_node # type: ZfsNode self.name = name # full actual name of dataset self.force_exists = force_exists @@ -179,7 +178,6 @@ class ZfsDataset: return self.__properties - def get_allowed_properties(self, filter_properties, set_properties): """only returns lists of allowed properties for this dataset type @@ -229,3 +227,24 @@ class ZfsDataset: # invalidate cache self.__properties = None + + def mount(self, mount_point): + """Mount the container or snapshot at mount_point, if it is a filesystem.""" + + self.debug("Mounting") + + cmd = [ + "mount", "-tzfs", self.name, mount_point + ] + + self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) + + def unmount(self, mount_point): + + self.debug("Unmounting") + + cmd = [ + "umount", mount_point + ] + + self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) diff --git a/zfs_autobackup/util.py b/zfs_autobackup/util.py index 76d75ab..c5e6532 100644 --- a/zfs_autobackup/util.py +++ b/zfs_autobackup/util.py @@ -18,6 +18,9 @@ def tmp_name(suffix=""): def get_tmp_clone_name(snapshot): + """ + :type snapshot: ZfsSnapshot + """ pool = snapshot.zfs_node.get_pool(snapshot) return pool.name + "/" + tmp_name() From 647aee01aa87cc6f6457424e45102fd3b5a5bb4a Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 11 Nov 2025 20:52:33 +0100 Subject: [PATCH 077/135] fix --- tests/test_externalfailures.py | 3 ++- tests/test_zfsautobackup40.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_externalfailures.py b/tests/test_externalfailures.py index 58d26c9..92f20c9 100644 --- a/tests/test_externalfailures.py +++ b/tests/test_externalfailures.py @@ -69,7 +69,8 @@ test_target1/test_source2/fs2/sub@test-20101111000000 def test_incremental_resume(self): # initial backup with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --keep-source=0".split(" ")).run()) # incremental backup leaves resume token with mocktime("20101111000001"): diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py index 07d07c6..fbb03f8 100644 --- a/tests/test_zfsautobackup40.py +++ b/tests/test_zfsautobackup40.py @@ -560,5 +560,6 @@ test_target1/test_source1/fs1/sub@test2-20010203000000 # TODO # - check if holds and no-holds function ok when bookmarks are enabled +# - does it remove hold when going from no-bookmark to bookmark mode? # - delete bookmarks we have on other snapshots # - test if resuming works with bookmarks From 9110389f5baf54d639e4950c16a0736f596a8bf7 Mon Sep 17 00:00:00 2001 From: DatuX Date: Tue, 11 Nov 2025 20:38:38 +0100 Subject: [PATCH 078/135] bump minimum version of python to 3.2, which has argparse built in --- requirements.txt | 1 - setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2319de0..b6bc9a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ colorama -argparse coverage python-coveralls unittest2 diff --git a/setup.py b/setup.py index 5ebd527..8547daa 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ setuptools.setup( "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", ], - python_requires='>=3.0', + python_requires='>=3.2', install_requires=[ "colorama", "argparse" From 78e4615f0b8c447e0f0eece62557a13f7432ad45 Mon Sep 17 00:00:00 2001 From: DatuX Date: Tue, 11 Nov 2025 23:02:01 +0100 Subject: [PATCH 079/135] run tests remote --- tests/autoremote | 3 +++ tests/remoteruntest | 10 ++++++++++ 2 files changed, 13 insertions(+) create mode 100755 tests/autoremote create mode 100755 tests/remoteruntest diff --git a/tests/autoremote b/tests/autoremote new file mode 100755 index 0000000..bef01bb --- /dev/null +++ b/tests/autoremote @@ -0,0 +1,3 @@ +#!/bin/bash + +find tests/*.py zfs_autobackup/*.py | entr -r ./tests/remoteruntest $@ diff --git a/tests/remoteruntest b/tests/remoteruntest new file mode 100755 index 0000000..f24208e --- /dev/null +++ b/tests/remoteruntest @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +HOST=root@ws2 +DST=/home/psy/zfs_autobackup + +rsync --delete -avx . "$HOST:$DST" + +ssh $HOST "cd $DST && ./tests/run_test $@" From 6e1b2bf2f871088f1e98b71c30379f853bc75d17 Mon Sep 17 00:00:00 2001 From: DatuX Date: Tue, 11 Nov 2025 23:24:28 +0100 Subject: [PATCH 080/135] run tests remote --- tests/autoremote | 1 + tests/run_test | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/autoremote b/tests/autoremote index bef01bb..c41e7e9 100755 --- a/tests/autoremote +++ b/tests/autoremote @@ -1,3 +1,4 @@ #!/bin/bash +# to run one: -k test_incremental_resume find tests/*.py zfs_autobackup/*.py | entr -r ./tests/remoteruntest $@ diff --git a/tests/run_test b/tests/run_test index 1824a5e..15ffdfc 100755 --- a/tests/run_test +++ b/tests/run_test @@ -1,5 +1,5 @@ #!/bin/bash -#run one test. start from main directory - +#run all or one test. start from main directory +# to run one: -k test_incremental_resume python -m unittest discover tests $@ -vvvf From 20190a3581f536c46369375678bcd3adc0ae3d47 Mon Sep 17 00:00:00 2001 From: DatuX Date: Tue, 11 Nov 2025 23:30:58 +0100 Subject: [PATCH 081/135] fix --- tests/test_externalfailures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_externalfailures.py b/tests/test_externalfailures.py index 92f20c9..9f27c92 100644 --- a/tests/test_externalfailures.py +++ b/tests/test_externalfailures.py @@ -70,7 +70,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # initial backup with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --allow-empty --keep-source=0".split(" ")).run()) + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) # incremental backup leaves resume token with mocktime("20101111000001"): From f144e6b4f5e616090b84e3da97a8bef43ea16269 Mon Sep 17 00:00:00 2001 From: DatuX Date: Wed, 12 Nov 2025 00:00:05 +0100 Subject: [PATCH 082/135] doc --- zfs_autobackup/ZfsDataset.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 9d04398..5beffab 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1,5 +1,31 @@ from .ExecuteNode import ExecuteError +# +# Inheritance / overview graph (comment): +# +# ZfsDataset +# ├─ ZfsPointInTime (zfs_autobackup/ZfsPointInTime.py) +# │ ├─ ZfsSnapshot (zfs_autobackup/ZfsSnapshot.py) +# │ └─ ZfsBookmark (zfs_autobackup/ZfsBookmark.py) +# └─ ZfsContainer (zfs_autobackup/ZfsContainer.py) +# +# Summary (key responsibilities / notable members): +# - ZfsDataset (this file) +# - Base for all dataset types (filesystems, volumes, snapshots, bookmarks) +# - Key attributes: .zfs_node, .name, .force_exists +# - Important properties: .exists_check, .exists, .properties +# - Important methods: destroy(), invalidate_cache(), set(), inherit(), mount(), unmount() +# - ZfsPointInTime +# - Abstract intermediate for point-in-time objects (snapshots & bookmarks) +# - Key: .prefix, .suffix, .timestamp, .is_ours, .tagless_suffix, .tag +# - ZfsSnapshot +# - Snapshot-specific operations: send_pipe(), recv_pipe(), transfer_snapshot(), bookmark(), clone(), hold/release +# - ZfsBookmark +# - Bookmark-specific behavior (name split on "#", light destroy override) +# - ZfsContainer +# - Filesystem/volume container logic: snapshots/bookmarks management, automount, thin/thin_list, sync_snapshots, find_common_snapshot +## + class ZfsDataset: """A generic ZFS dataset, this has all the common functions of zfs filesystems, volumes, snapshots and bookmarks. From f046f309ff983b3db9f796aab362ecaf7c0d713d Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 00:33:03 +0200 Subject: [PATCH 083/135] resuming with bookmark is already tested in the default test; --- tests/test_externalfailures.py | 6 +++--- tests/test_zfsautobackup40.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_externalfailures.py b/tests/test_externalfailures.py index 9f27c92..742818d 100644 --- a/tests/test_externalfailures.py +++ b/tests/test_externalfailures.py @@ -33,7 +33,7 @@ class TestExternalFailures(unittest2.TestCase): with mocktime("20101111000000"): self.generate_resume() - # --test should resume and succeed + # test should resume and succeed with OutputIO() as buf: with redirect_stdout(buf): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test".split(" ")).run()) @@ -111,7 +111,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 """) # generate an invalid resume token, and verify if its aborted automaticly - def test_initial_resumeabort(self): + def test_initial_abortedresume(self): # inital backup, leaves resume token with mocktime("20101111000000"): self.generate_resume() @@ -145,7 +145,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 """) # generate an invalid resume token, and verify if its aborted automaticly - def test_incremental_resumeabort(self): + def test_incremental_abortedresume(self): # initial backup with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py index fbb03f8..9d9ea6b 100644 --- a/tests/test_zfsautobackup40.py +++ b/tests/test_zfsautobackup40.py @@ -558,8 +558,8 @@ test_target1/test_source1/fs1/sub@other2 test_target1/test_source1/fs1/sub@test2-20010203000000 """) + # TODO # - check if holds and no-holds function ok when bookmarks are enabled # - does it remove hold when going from no-bookmark to bookmark mode? -# - delete bookmarks we have on other snapshots -# - test if resuming works with bookmarks +# - delete bookmarks we have no other snapshots From baf9681de91b1bb8ff89960e13bdb16a86946149 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 00:38:21 +0200 Subject: [PATCH 084/135] lets start beta testing :) --- zfs_autobackup/CliBase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zfs_autobackup/CliBase.py b/zfs_autobackup/CliBase.py index 6be5424..e3444c8 100644 --- a/zfs_autobackup/CliBase.py +++ b/zfs_autobackup/CliBase.py @@ -10,8 +10,8 @@ class CliBase(object): Overridden in subclasses that add stuff for the specific programs.""" # also used by setup.py - VERSION = "4.0-alpha" - HEADER = "{} v{} - (c)2022 E.H.Eefting (edwin@datux.nl)".format(os.path.basename(sys.argv[0]), VERSION) + VERSION = "4.0-beta1" + HEADER = "{} v{} - (c)2026 E.H.Eefting (edwin@datux.nl)".format(os.path.basename(sys.argv[0]), VERSION) def __init__(self, argv, print_arguments=True): From 29459afc65edf3a45ee94b574dcd215868eea56c Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 00:49:22 +0200 Subject: [PATCH 085/135] you actually dont have to delete the dataset and childeren if there is no common snapshot --- zfs_autobackup/ZfsContainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index d9f2df7..17c00bb 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -499,7 +499,7 @@ class ZfsContainer(ZfsDataset): if not self.zfs_node.readonly: raise e - raise (Exception("Cant find common bookmark or snapshot with target.")) + raise (Exception("Cant find common bookmark or snapshot with target. (Delete snapshots on target and use -F to fix this.)")) def _pre_clean(self, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): """cleanup old stuff on the source before starting snapshot syncing From 2c0b0668f6ed906da3ad987647de1862495bca70 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 01:27:47 +0200 Subject: [PATCH 086/135] fix #107 --- tests/test_zfsautobackup.py | 96 +++++++++++++++++++++++++++++++++++++ zfs_autobackup/ZfsAuto.py | 13 +++-- 2 files changed, 106 insertions(+), 3 deletions(-) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 26b34a8..2117226 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -426,6 +426,102 @@ test_target1/fs2/sub@test-20101111000000 with self.assertRaisesRegexp(Exception, "too much"): ZfsAutobackup("test test_target1 --verbose --strip-path=3 --no-progress --debug".split(" ")).run() + def test_property_format(self): + """--property-format selects datasets via a custom ZFS user-property name. + + The literal "{}" in the format is substituted with BACKUP-NAME, producing + the actual property name that zfs-autobackup looks for on each dataset. + """ + + # the default "autobackup:test" property on test_source1/fs1 and test_source2/fs2 + # should be ignored once we change --property-format. Set a property using the + # custom format on test_source2/fs3 instead, which is normally not selected. + shelltest("zfs set mybackup:test=true test_source2/fs3") + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --property-format mybackup:{}".split(" ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1/sub +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs3 +test_source2/fs3@test-20101111000000 +test_source2/fs3/sub +test_source2/fs3/sub@test-20101111000000 +test_target1 +test_target1/test_source2 +test_target1/test_source2/fs3 +test_target1/test_source2/fs3@test-20101111000000 +test_target1/test_source2/fs3/sub +test_target1/test_source2/fs3/sub@test-20101111000000 +""") + + def test_snapshot_format(self): + """--snapshot-format controls the snapshot name after the "@". + + "{}" is substituted with BACKUP-NAME, then strftime codes are expanded + against the current time. The result must not contain the tag-seperator. + """ + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --snapshot-format snap-{}-%Y-%m-%d".split(" ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@snap-test-2010-11-11 +test_source1/fs1/sub +test_source1/fs1/sub@snap-test-2010-11-11 +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@snap-test-2010-11-11 +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@snap-test-2010-11-11 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@snap-test-2010-11-11 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@snap-test-2010-11-11 +""") + + def test_snapshot_format_tagseperator_collision(self): + """If --snapshot-format produces a name that contains the tag-seperator, refuse to run.""" + + with OutputIO() as buf: + with redirect_stderr(buf): + with mocktime("20101111000000"): + # default tag-seperator "_" appears in the strftime output -> sys.exit(255) + with self.assertRaises(SystemExit) as cm: + ZfsAutobackup("test test_target1 --no-progress --snapshot-format {}_%Y_%m_%d".split(" ")) + self.assertEqual(cm.exception.code, 255) + + self.assertIn("Tag seperator", buf.getvalue()) + + def test_hold_format(self): + """--hold-format controls the zfs-hold name placed on snapshots.""" + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-bookmark --hold-format myhold_{}".split(" ")).run()) + + r = shelltest("zfs holds -H test_source1/fs1@test-20101111000000") + self.assertIn("myhold_test", r) + self.assertNotIn("zfs_autobackup:test", r) + def test_clearrefres(self): # on zfs utils 0.6.x -x isnt supported diff --git a/zfs_autobackup/ZfsAuto.py b/zfs_autobackup/ZfsAuto.py index d03b124..7d7595a 100644 --- a/zfs_autobackup/ZfsAuto.py +++ b/zfs_autobackup/ZfsAuto.py @@ -113,11 +113,18 @@ class ZfsAuto(CliBase): group = parser.add_argument_group("String formatting options") group.add_argument('--property-format', metavar='FORMAT', default="autobackup:{}", - help='Dataset selection string format. Default: %(default)s') + help='Name of the ZFS user-property used to select datasets for this backup. ' + 'The literal "{}" is substituted with BACKUP-NAME. ' + 'Default: %(default)s') group.add_argument('--snapshot-format', metavar='FORMAT', default="{}-%Y%m%d%H%M%S", - help='ZFS Snapshot string format. Default: %(default)s') + help='Format of the snapshot-name part after the "@". ' + 'The literal "{}" is substituted with BACKUP-NAME, then strftime-codes ' + '(e.g. %%Y %%m %%d %%H %%M %%S) are expanded against the current time. ' + 'Must not contain the tag-seperator. Default: %(default)s') group.add_argument('--hold-format', metavar='FORMAT', default="zfs_autobackup:{}", - help='ZFS hold string format. Default: %(default)s') + help='Name of the zfs-hold placed on snapshots to prevent accidental deletion. ' + 'The literal "{}" is substituted with BACKUP-NAME. ' + 'Default: %(default)s') group.add_argument('--strip-path', metavar='N', default=0, type=int, help='Number of directories to strip from target path.') group.add_argument('--tag-seperator', metavar='CHAR', default="_", From 80f7c4b22b487fd69cc060e338958bd0aa806670 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 10:02:58 +0200 Subject: [PATCH 087/135] fixes #325 --- tests/test_zfsautobackup.py | 2 +- tests/test_zfsautobackup40.py | 104 +++++++++++++++++----------------- tests/test_zfscheck.py | 2 +- zfs_autobackup/ZfsAuto.py | 6 +- zfs_autobackup/ZfsCheck.py | 2 +- 5 files changed, 58 insertions(+), 58 deletions(-) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 2117226..2451152 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -506,7 +506,7 @@ test_target1/test_source2/fs2/sub@snap-test-2010-11-11 with mocktime("20101111000000"): # default tag-seperator "_" appears in the strftime output -> sys.exit(255) with self.assertRaises(SystemExit) as cm: - ZfsAutobackup("test test_target1 --no-progress --snapshot-format {}_%Y_%m_%d".split(" ")) + ZfsAutobackup("test test_target1 --no-progress --snapshot-format {}__%Y_%m_%d".split(" ")) self.assertEqual(cm.exception.code, 255) self.assertIn("Tag seperator", buf.getvalue()) diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py index 9d9ea6b..791d8b3 100644 --- a/tests/test_zfsautobackup40.py +++ b/tests/test_zfsautobackup40.py @@ -48,13 +48,13 @@ class TestZfsAutobackup40(unittest2.TestCase): # destroy stuff and see if it still selects the correct ones shelltest("zfs destroy test_source2/fs2/sub@test-20101111000001") - shelltest("zfs destroy test_source1/fs1/sub#test-20101111000001_" + target_guid) + shelltest("zfs destroy test_source1/fs1/sub#test-20101111000001__" + target_guid) # replace bookmark with incorrect GUID, should fallback to snapshot # NOTE: the incorrect bookmark should be ignored and not destroyed. - shelltest("zfs destroy test_source1/fs1#test-20101111000001_" + target_guid) + shelltest("zfs destroy test_source1/fs1#test-20101111000001__" + target_guid) shelltest("zfs snapshot test_source1/fs1@wrong") - shelltest("zfs bookmark test_source1/fs1@wrong \#test-20101111000001_" + target_guid) + shelltest("zfs bookmark test_source1/fs1@wrong \#test-20101111000001__" + target_guid) shelltest("zfs destroy test_source1/fs1@wrong") with mocktime("20101111000002"): @@ -66,17 +66,17 @@ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000001 test_source1/fs1@test-20101111000002 -test_source1/fs1#test-20101111000001_[0-9]* -test_source1/fs1#test-20101111000002_[0-9]* +test_source1/fs1#test-20101111000001__[0-9]* +test_source1/fs1#test-20101111000002__[0-9]* test_source1/fs1/sub test_source1/fs1/sub@test-20101111000001 test_source1/fs1/sub@test-20101111000002 -test_source1/fs1/sub#test-20101111000002_[0-9]* +test_source1/fs1/sub#test-20101111000002__[0-9]* test_source2 test_source2/fs2 test_source2/fs2/sub test_source2/fs2/sub@test-20101111000002 -test_source2/fs2/sub#test-20101111000002_[0-9]* +test_source2/fs2/sub#test-20101111000002__[0-9]* test_source2/fs3 test_source2/fs3/sub test_target1 @@ -110,10 +110,10 @@ test_target1/test_source2/fs2/sub@test-20101111000002 test_source1 test_source1/fs1 test_source1/fs1@test-20101111000001 -test_source1/fs1#test-20101111000001_[0-9]* +test_source1/fs1#test-20101111000001__[0-9]* test_source1/fs1/sub test_source1/fs1/sub@test-20101111000001 -test_source1/fs1/sub#test-20101111000001_[0-9]* +test_source1/fs1/sub#test-20101111000001__[0-9]* """) # disable it (this should not touch the old bookmark) @@ -127,11 +127,11 @@ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000001 test_source1/fs1@test-20101111000002 -test_source1/fs1#test-20101111000001_[0-9]* +test_source1/fs1#test-20101111000001__[0-9]* test_source1/fs1/sub test_source1/fs1/sub@test-20101111000001 test_source1/fs1/sub@test-20101111000002 -test_source1/fs1/sub#test-20101111000001_[0-9]* +test_source1/fs1/sub#test-20101111000001__[0-9]* """) # re-enable (now the old bookmark should be still left alone) @@ -145,14 +145,14 @@ test_source1/fs1 test_source1/fs1@test-20101111000001 test_source1/fs1@test-20101111000002 test_source1/fs1@test-20101111000003 -test_source1/fs1#test-20101111000001_[0-9]* -test_source1/fs1#test-20101111000003_[0-9]* +test_source1/fs1#test-20101111000001__[0-9]* +test_source1/fs1#test-20101111000003__[0-9]* test_source1/fs1/sub test_source1/fs1/sub@test-20101111000001 test_source1/fs1/sub@test-20101111000002 test_source1/fs1/sub@test-20101111000003 -test_source1/fs1/sub#test-20101111000001_[0-9]* -test_source1/fs1/sub#test-20101111000003_[0-9]* +test_source1/fs1/sub#test-20101111000001__[0-9]* +test_source1/fs1/sub#test-20101111000003__[0-9]* """) def test_tags(self): @@ -173,17 +173,17 @@ test_source1/fs1/sub#test-20101111000003_[0-9]* r = shelltest("zfs list -H -r -t snapshot -o name " + TEST_POOLS) self.assertMultiLineEqual(r, """ -test_source1/fs1@test-20101111000002_test2 +test_source1/fs1@test-20101111000002__test2 test_source1/fs1@test-20101111000003 -test_source1/fs1/sub@test-20101111000002_test2 +test_source1/fs1/sub@test-20101111000002__test2 test_source1/fs1/sub@test-20101111000003 -test_source2/fs2/sub@test-20101111000002_test2 +test_source2/fs2/sub@test-20101111000002__test2 test_source2/fs2/sub@test-20101111000003 -test_target1/test_source1/fs1@test-20101111000002_test2 +test_target1/test_source1/fs1@test-20101111000002__test2 test_target1/test_source1/fs1@test-20101111000003 -test_target1/test_source1/fs1/sub@test-20101111000002_test2 +test_target1/test_source1/fs1/sub@test-20101111000002__test2 test_target1/test_source1/fs1/sub@test-20101111000003 -test_target1/test_source2/fs2/sub@test-20101111000002_test2 +test_target1/test_source2/fs2/sub@test-20101111000002__test2 test_target1/test_source2/fs2/sub@test-20101111000003 """) @@ -219,56 +219,56 @@ test_target1/test_source2/fs2/sub@test-20101111000003 self.assertRegexpMatches(r, """ test_source1 test_source1/fs1 -test_source1/fs1@test-20101111000001_tagA -test_source1/fs1@test-20101111000002_tagB +test_source1/fs1@test-20101111000001__tagA +test_source1/fs1@test-20101111000002__tagB test_source1/fs1@test-20101111000003 -test_source1/fs1#test-20101111000002_[0-9]* -test_source1/fs1#test-20101111000003_[0-9]* +test_source1/fs1#test-20101111000002__[0-9]* +test_source1/fs1#test-20101111000003__[0-9]* test_source1/fs1/sub -test_source1/fs1/sub@test-20101111000001_tagA -test_source1/fs1/sub@test-20101111000002_tagB +test_source1/fs1/sub@test-20101111000001__tagA +test_source1/fs1/sub@test-20101111000002__tagB test_source1/fs1/sub@test-20101111000003 -test_source1/fs1/sub#test-20101111000002_[0-9]* -test_source1/fs1/sub#test-20101111000003_[0-9]* +test_source1/fs1/sub#test-20101111000002__[0-9]* +test_source1/fs1/sub#test-20101111000003__[0-9]* test_source2 test_source2/fs2 test_source2/fs2/sub -test_source2/fs2/sub@test-20101111000001_tagA -test_source2/fs2/sub@test-20101111000002_tagB +test_source2/fs2/sub@test-20101111000001__tagA +test_source2/fs2/sub@test-20101111000002__tagB test_source2/fs2/sub@test-20101111000003 -test_source2/fs2/sub#test-20101111000002_[0-9]* -test_source2/fs2/sub#test-20101111000003_[0-9]* +test_source2/fs2/sub#test-20101111000002__[0-9]* +test_source2/fs2/sub#test-20101111000003__[0-9]* test_source2/fs3 test_source2/fs3/sub test_target1 test_target1/a test_target1/a/test_source1 test_target1/a/test_source1/fs1 -test_target1/a/test_source1/fs1@test-20101111000001_tagA -test_target1/a/test_source1/fs1@test-20101111000002_tagB +test_target1/a/test_source1/fs1@test-20101111000001__tagA +test_target1/a/test_source1/fs1@test-20101111000002__tagB test_target1/a/test_source1/fs1/sub -test_target1/a/test_source1/fs1/sub@test-20101111000001_tagA -test_target1/a/test_source1/fs1/sub@test-20101111000002_tagB +test_target1/a/test_source1/fs1/sub@test-20101111000001__tagA +test_target1/a/test_source1/fs1/sub@test-20101111000002__tagB test_target1/a/test_source2 test_target1/a/test_source2/fs2 test_target1/a/test_source2/fs2/sub -test_target1/a/test_source2/fs2/sub@test-20101111000001_tagA -test_target1/a/test_source2/fs2/sub@test-20101111000002_tagB +test_target1/a/test_source2/fs2/sub@test-20101111000001__tagA +test_target1/a/test_source2/fs2/sub@test-20101111000002__tagB test_target1/b test_target1/b/test_source1 test_target1/b/test_source1/fs1 -test_target1/b/test_source1/fs1@test-20101111000001_tagA -test_target1/b/test_source1/fs1@test-20101111000002_tagB +test_target1/b/test_source1/fs1@test-20101111000001__tagA +test_target1/b/test_source1/fs1@test-20101111000002__tagB test_target1/b/test_source1/fs1@test-20101111000003 test_target1/b/test_source1/fs1/sub -test_target1/b/test_source1/fs1/sub@test-20101111000001_tagA -test_target1/b/test_source1/fs1/sub@test-20101111000002_tagB +test_target1/b/test_source1/fs1/sub@test-20101111000001__tagA +test_target1/b/test_source1/fs1/sub@test-20101111000002__tagB test_target1/b/test_source1/fs1/sub@test-20101111000003 test_target1/b/test_source2 test_target1/b/test_source2/fs2 test_target1/b/test_source2/fs2/sub -test_target1/b/test_source2/fs2/sub@test-20101111000001_tagA -test_target1/b/test_source2/fs2/sub@test-20101111000002_tagB +test_target1/b/test_source2/fs2/sub@test-20101111000001__tagA +test_target1/b/test_source2/fs2/sub@test-20101111000002__tagB test_target1/b/test_source2/fs2/sub@test-20101111000003 """) @@ -443,15 +443,15 @@ test_target1/test_source2/fs2/sub@test-20101111000000 test_source1 test_source1/fs1 test_source1/fs1@test-20101111000002 -test_source1/fs1#test-20101111000000_[0-9]* +test_source1/fs1#test-20101111000000__[0-9]* test_source1/fs1/sub test_source1/fs1/sub@test-20101111000002 -test_source1/fs1/sub#test-20101111000000_[0-9]* +test_source1/fs1/sub#test-20101111000000__[0-9]* test_source2 test_source2/fs2 test_source2/fs2/sub test_source2/fs2/sub@test-20101111000002 -test_source2/fs2/sub#test-20101111000000_[0-9]* +test_source2/fs2/sub#test-20101111000000__[0-9]* test_source2/fs3 test_source2/fs3/sub test_target1 @@ -476,13 +476,13 @@ test_target1/test_source2/fs2/sub@test-20101111000000 self.assertRegexpMatches(r, """ test_source1 test_source1/fs1 -test_source1/fs1#test-20101111000003_[0-9]* +test_source1/fs1#test-20101111000003__[0-9]* test_source1/fs1/sub -test_source1/fs1/sub#test-20101111000003_[0-9]* +test_source1/fs1/sub#test-20101111000003__[0-9]* test_source2 test_source2/fs2 test_source2/fs2/sub -test_source2/fs2/sub#test-20101111000003_[0-9]* +test_source2/fs2/sub#test-20101111000003__[0-9]* test_source2/fs3 test_source2/fs3/sub test_target1 @@ -550,7 +550,7 @@ test_source1/fs1/sub@test2-20010201000000 test_source1/fs1/sub@other2 test_source1/fs1/sub@test2-20010202000000 test_source1/fs1/sub@test2-20010203000000 -test_source1/fs1/sub#test2-20010203000000_[0-9]* +test_source1/fs1/sub#test2-20010203000000__[0-9]* test_target1/test_source1/fs1/sub@test2-20010101000000 test_target1/test_source1/fs1/sub@other1 test_target1/test_source1/fs1/sub@test2-20010201000000 diff --git a/tests/test_zfscheck.py b/tests/test_zfscheck.py index d836208..de4370f 100644 --- a/tests/test_zfscheck.py +++ b/tests/test_zfscheck.py @@ -209,7 +209,7 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bd8 # breaks pipe when grep exists: # important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling - shelltest("python -m zfs_autobackup.ZfsCheck test_source1/vol@test --debug| grep -m1 'Hashing file'") + shelltest("python3 -m zfs_autobackup.ZfsCheck test_source1/vol@test --debug| grep -m1 'Hashing file'") # time.sleep(1) r = shelltest("zfs list -H -o name -r -t snapshot,filesystem,volume " + TEST_POOLS) diff --git a/zfs_autobackup/ZfsAuto.py b/zfs_autobackup/ZfsAuto.py index 7d7595a..958b901 100644 --- a/zfs_autobackup/ZfsAuto.py +++ b/zfs_autobackup/ZfsAuto.py @@ -73,8 +73,8 @@ class ZfsAuto(CliBase): # according to man 8 zfs: valid_tags = "_.: -" - if self.tag_seperator not in valid_tags or self.tag_seperator == '': - self.log.error("Invalid tag seperator. Allowed: '{}'".format(valid_tags)) + if self.tag_seperator == '' or any(c not in valid_tags for c in self.tag_seperator): + self.log.error("Invalid tag seperator. Allowed characters: '{}'".format(valid_tags)) sys.exit(255) if self.tag_seperator in seperator_test: @@ -127,7 +127,7 @@ class ZfsAuto(CliBase): 'Default: %(default)s') group.add_argument('--strip-path', metavar='N', default=0, type=int, help='Number of directories to strip from target path.') - group.add_argument('--tag-seperator', metavar='CHAR', default="_", + group.add_argument('--tag-seperator', metavar='STRING', default="__", help="Tag seperator for snapshots and bookmarks. Default: %(default)s") group.add_argument('--tag', metavar='TAG', default=None, help='Backup tag to add to snapshots names. (For administrative purposes)') diff --git a/zfs_autobackup/ZfsCheck.py b/zfs_autobackup/ZfsCheck.py index 68048d5..f1254ac 100644 --- a/zfs_autobackup/ZfsCheck.py +++ b/zfs_autobackup/ZfsCheck.py @@ -22,7 +22,7 @@ class ZfsCheck(CliBase): self.node = ZfsNode(self.log, utc=self.args.utc, readonly=self.args.test, debug_output=self.args.debug_output, ssh_to=None, ssh_config=None, exclude_snapshot_patterns=None, thinner=None, - tag_seperator='_', description="", hold_name=None, snapshot_time_format=None) + tag_seperator='__', description="", hold_name=None, snapshot_time_format=None) self.block_hasher = BlockHasher(count=self.args.count, bs=self.args.block_size, skip=self.args.skip) From fe0111259a717564ff886c6011e0deed95c8d096 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 10:40:26 +0200 Subject: [PATCH 088/135] fixed unittest so that it can run one test from pycharm. removed recursive symlink. add tests for tag seperator --- .gitignore | 1 + tests/__init__.py | 6 ++++++ tests/test_blockhasher.py | 40 ++++++++++++++++++------------------- tests/test_treehasher.py | 6 +++--- tests/test_zfsautobackup.py | 29 ++++++++++++++++++++++++++- tests/test_zfscheck.py | 20 +++++++++---------- tests/tests | 1 - 7 files changed, 68 insertions(+), 35 deletions(-) create mode 100644 tests/__init__.py delete mode 120000 tests/tests diff --git a/.gitignore b/.gitignore index 723d68a..df2d0e8 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ python2.env venv .idea password.sh +.DS_Store diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..66b16a7 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,6 @@ +import os +import sys + +_here = os.path.dirname(os.path.abspath(__file__)) +if _here not in sys.path: + sys.path.insert(0, _here) diff --git a/tests/test_blockhasher.py b/tests/test_blockhasher.py index 957d487..3214d6b 100644 --- a/tests/test_blockhasher.py +++ b/tests/test_blockhasher.py @@ -20,28 +20,28 @@ class TestBlockHasher(unittest2.TestCase): def test_empty(self): block_hasher = BlockHasher(count=1) self.assertEqual( - list(block_hasher.generate("tests/data/empty")), + list(block_hasher.generate("data/empty")), [] ) def test_partial(self): block_hasher = BlockHasher(count=1) self.assertEqual( - list(block_hasher.generate("tests/data/partial")), + list(block_hasher.generate("data/partial")), [(0, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1")] ) def test_whole(self): block_hasher = BlockHasher(count=1) self.assertEqual( - list(block_hasher.generate("tests/data/whole")), + list(block_hasher.generate("data/whole")), [(0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7")] ) def test_whole2(self): block_hasher = BlockHasher(count=1) self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2")), + list(block_hasher.generate("data/whole_whole2")), [ (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798") @@ -51,7 +51,7 @@ class TestBlockHasher(unittest2.TestCase): def test_wwp(self): block_hasher = BlockHasher(count=1) self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), # whole (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), # whole2 @@ -62,7 +62,7 @@ class TestBlockHasher(unittest2.TestCase): def test_wwp_count2(self): block_hasher = BlockHasher(count=2) self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ (0, "959e6b58078f0cfd2fb3d37e978fda51820473ff"), # whole_whole2 (1, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1") # partial @@ -72,7 +72,7 @@ class TestBlockHasher(unittest2.TestCase): def test_big(self): block_hasher = BlockHasher(count=10) self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ (0, "309ffffba2e1977d12f3b7469971f30d28b94bd8"), # whole_whole2_partial ]) @@ -80,32 +80,32 @@ class TestBlockHasher(unittest2.TestCase): def test_blockhash_compare(self): #no errors block_hasher = BlockHasher(count=1) - generator = block_hasher.generate("tests/data/whole_whole2_partial") - self.assertEqual([], list(block_hasher.compare("tests/data/whole_whole2_partial", generator))) + generator = block_hasher.generate("data/whole_whole2_partial") + self.assertEqual([], list(block_hasher.compare("data/whole_whole2_partial", generator))) #compare file is smaller (EOF errors) block_hasher = BlockHasher(count=1) - generator = block_hasher.generate("tests/data/whole_whole2_partial") + generator = block_hasher.generate("data/whole_whole2_partial") self.assertEqual( [(1, '2e863f1fcccd6642e4e28453eba10d2d3f74d798', 'EOF'), (2, '642027d63bb0afd7e0ba197f2c66ad03e3d70de1', 'EOF')], - list(block_hasher.compare("tests/data/whole", generator))) + list(block_hasher.compare("data/whole", generator))) #no errors, huge chunks block_hasher = BlockHasher(count=10) - generator = block_hasher.generate("tests/data/whole_whole2_partial") - self.assertEqual([], list(block_hasher.compare("tests/data/whole_whole2_partial", generator))) + generator = block_hasher.generate("data/whole_whole2_partial") + self.assertEqual([], list(block_hasher.compare("data/whole_whole2_partial", generator))) # different order to make sure seek functions are ok block_hasher = BlockHasher(count=1) - checksums = list(block_hasher.generate("tests/data/whole_whole2_partial")) + checksums = list(block_hasher.generate("data/whole_whole2_partial")) checksums.reverse() - self.assertEqual([], list(block_hasher.compare("tests/data/whole_whole2_partial", checksums))) + self.assertEqual([], list(block_hasher.compare("data/whole_whole2_partial", checksums))) def test_skip1(self): block_hasher = BlockHasher(count=1, skip=1) self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), # whole # (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), # whole2 @@ -115,7 +115,7 @@ class TestBlockHasher(unittest2.TestCase): #should continue the pattern on the next file: self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ # (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), # whole (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), # whole2 @@ -126,7 +126,7 @@ class TestBlockHasher(unittest2.TestCase): def test_skip6(self): block_hasher = BlockHasher(count=1, skip=6) self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), # whole # (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), # whole2 @@ -136,7 +136,7 @@ class TestBlockHasher(unittest2.TestCase): #all blocks of next file are skipped self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ # (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), # whole # (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), # whole2 @@ -146,7 +146,7 @@ class TestBlockHasher(unittest2.TestCase): #first block of this one is the 6th to be skipped: self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ # (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), # whole (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), # whole2 diff --git a/tests/test_treehasher.py b/tests/test_treehasher.py index ea3cd04..e4648d6 100644 --- a/tests/test_treehasher.py +++ b/tests/test_treehasher.py @@ -15,10 +15,10 @@ class TestTreeHasher(unittest2.TestCase): def test_treehasher(self): shelltest("rm -rf /tmp/treehashertest; mkdir /tmp/treehashertest") - shelltest("cp tests/data/whole /tmp/treehashertest") + shelltest("cp data/whole /tmp/treehashertest") shelltest("mkdir /tmp/treehashertest/emptydir") shelltest("mkdir /tmp/treehashertest/dir") - shelltest("cp tests/data/whole_whole2_partial /tmp/treehashertest/dir") + shelltest("cp data/whole_whole2_partial /tmp/treehashertest/dir") # it should ignore these: shelltest("ln -s / /tmp/treehashertest/symlink") @@ -66,7 +66,7 @@ class TestTreeHasher(unittest2.TestCase): with self.subTest("Test mismatch"): generator = list(tree_hasher.generate("/tmp/treehashertest")) - shelltest("cp tests/data/whole2 /tmp/treehashertest/whole") + shelltest("cp data/whole2 /tmp/treehashertest/whole") self.assertEqual(list(tree_hasher.compare("/tmp/treehashertest", generator)), [('whole', diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 2451152..0a54245 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -504,13 +504,40 @@ test_target1/test_source2/fs2/sub@snap-test-2010-11-11 with OutputIO() as buf: with redirect_stderr(buf): with mocktime("20101111000000"): - # default tag-seperator "_" appears in the strftime output -> sys.exit(255) + # rendered snapshot-format "test__2010_11_11" contains default tag-seperator "__" -> sys.exit(255) with self.assertRaises(SystemExit) as cm: ZfsAutobackup("test test_target1 --no-progress --snapshot-format {}__%Y_%m_%d".split(" ")) self.assertEqual(cm.exception.code, 255) self.assertIn("Tag seperator", buf.getvalue()) + def test_tagseperator_invalid(self): + """--tag-seperator must be non-empty and contain only characters valid in ZFS snapshot names ('_.: -').""" + + # empty string + with OutputIO() as buf: + with redirect_stderr(buf): + with self.assertRaises(SystemExit) as cm: + ZfsAutobackup("test test_target1 --no-progress --tag-seperator=".split(" ")) + self.assertEqual(cm.exception.code, 255) + self.assertIn("Invalid tag seperator", buf.getvalue()) + + # single invalid character + with OutputIO() as buf: + with redirect_stderr(buf): + with self.assertRaises(SystemExit) as cm: + ZfsAutobackup("test test_target1 --no-progress --tag-seperator=!".split(" ")) + self.assertEqual(cm.exception.code, 255) + self.assertIn("Invalid tag seperator", buf.getvalue()) + + # multi-char string where one character is invalid + with OutputIO() as buf: + with redirect_stderr(buf): + with self.assertRaises(SystemExit) as cm: + ZfsAutobackup("test test_target1 --no-progress --tag-seperator=_!".split(" ")) + self.assertEqual(cm.exception.code, 255) + self.assertIn("Invalid tag seperator", buf.getvalue()) + def test_hold_format(self): """--hold-format controls the zfs-hold name placed on snapshots.""" diff --git a/tests/test_zfscheck.py b/tests/test_zfscheck.py index de4370f..b89eb09 100644 --- a/tests/test_zfscheck.py +++ b/tests/test_zfscheck.py @@ -46,10 +46,10 @@ class TestZfsCheck(unittest2.TestCase): def test_filesystem(self): prepare_zpools() - shelltest("cp tests/data/whole /test_source1/testfile") + shelltest("cp data/whole /test_source1/testfile") shelltest("mkdir /test_source1/emptydir") shelltest("mkdir /test_source1/dir") - shelltest("cp tests/data/whole2 /test_source1/dir/testfile") + shelltest("cp data/whole2 /test_source1/dir/testfile") # it should ignore these: shelltest("ln -s / /test_source1/symlink") @@ -89,7 +89,7 @@ dir/testfile 0 2e863f1fcccd6642e4e28453eba10d2d3f74d798 with self.subTest("Generate"): with OutputIO() as buf: with redirect_stdout(buf): - self.assertFalse(ZfsCheck("tests/data/whole".split(" "), print_arguments=False).run()) + self.assertFalse(ZfsCheck("data/whole".split(" "), print_arguments=False).run()) print(buf.getvalue()) self.assertEqual("""0 3c0bf91170d873b8e327d3bafb6bc074580d11b7 @@ -102,7 +102,7 @@ dir/testfile 0 2e863f1fcccd6642e4e28453eba10d2d3f74d798 with self.subTest("Compare"): with OutputIO() as buf: with redirect_stdout(buf): - self.assertEqual(1, ZfsCheck("tests/data/whole --check=/tmp/testhashes".split(" "), + self.assertEqual(1, ZfsCheck("data/whole --check=/tmp/testhashes".split(" "), print_arguments=False).run()) print(buf.getvalue()) self.assertEqual( @@ -111,11 +111,11 @@ dir/testfile 0 2e863f1fcccd6642e4e28453eba10d2d3f74d798 def test_tree(self): shelltest("rm -rf /tmp/testtree; mkdir /tmp/testtree") - shelltest("cp tests/data/whole /tmp/testtree") - shelltest("cp tests/data/whole_whole2 /tmp/testtree") - shelltest("cp tests/data/whole2 /tmp/testtree") - shelltest("cp tests/data/partial /tmp/testtree") - shelltest("cp tests/data/whole_whole2_partial /tmp/testtree") + shelltest("cp data/whole /tmp/testtree") + shelltest("cp data/whole_whole2 /tmp/testtree") + shelltest("cp data/whole2 /tmp/testtree") + shelltest("cp data/partial /tmp/testtree") + shelltest("cp data/whole_whole2_partial /tmp/testtree") #################################### with self.subTest("Generate, skip 1"): @@ -189,7 +189,7 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bd8 """test if stuff is cleaned up correctly, in debugging mode , when a pipe breaks. """ prepare_zpools() - shelltest("cp tests/data/whole /test_source1/testfile") + shelltest("cp data/whole /test_source1/testfile") shelltest("zfs snapshot test_source1@test") # breaks pipe when head exists diff --git a/tests/tests b/tests/tests deleted file mode 120000 index 945c9b4..0000000 --- a/tests/tests +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file From dddee1f52f71e4478daaa13b62f1c8b981597721 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 11:01:37 +0200 Subject: [PATCH 089/135] several small fixes and tests --- tests/test_executenode.py | 19 ++++++++++++++ tests/test_log.py | 18 +++++++++++++ tests/test_thinner.py | 45 +++++++++++++++++++++++++++++++++ tests/test_zfsautobackup.py | 34 +++++++++++++++++++++++++ zfs_autobackup/CmdPipe.py | 16 ++++++------ zfs_autobackup/ExecuteNode.py | 2 +- zfs_autobackup/LogConsole.py | 29 ++++++++++----------- zfs_autobackup/ZfsAutobackup.py | 2 +- zfs_autobackup/ZfsSnapshot.py | 4 +-- 9 files changed, 142 insertions(+), 27 deletions(-) diff --git a/tests/test_executenode.py b/tests/test_executenode.py index 1b1db1f..cee137c 100644 --- a/tests/test_executenode.py +++ b/tests/test_executenode.py @@ -202,6 +202,25 @@ class TestExecuteNode(unittest2.TestCase): self.assertEqual(result, ["tEst"]) + def test_script_remote_ssh_config_flat_argv(self): + """script() with ssh_to + ssh_config must build a flat argv (no nested lists). + + Regression: previously used cmd.append(["-F", ssh_config]) which embedded a + list inside the argv and broke subprocess.Popen when actually executing. + """ + + node = ExecuteNode(ssh_to="user@somehost", ssh_config="/tmp/some_ssh_config") + cmd_pipe = node.script(lines=["echo test"], pipe=True) + argv = cmd_pipe.items[0].cmd + + for i, arg in enumerate(argv): + self.assertIsInstance(arg, str, msg="argv[{}] should be str, got {!r}".format(i, arg)) + + # also verify the -F flag and value are present and adjacent + self.assertIn("-F", argv) + f_index = argv.index("-F") + self.assertEqual(argv[f_index + 1], "/tmp/some_ssh_config") + diff --git a/tests/test_log.py b/tests/test_log.py index ba788e2..e4540df 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -24,6 +24,24 @@ class TestLog(unittest2.TestCase): print(list(buf.getvalue())) self.assertEqual(list(buf.getvalue()), ['\x1b', '[', '2', '2', 'm', ' ', ' ', 'v', 'e', 'r', 'b', 'o', 's', 'e', '\x1b', '[', '0', 'm', '\n', '\x1b', '[', '3', '2', 'm', '#', ' ', 'd', 'e', 'b', 'u', 'g', '\x1b', '[', '0', 'm', '\n', '\x1b', '[', '3', '1', 'm', '\x1b', '[', '1', 'm', '!', ' ', 'e', 'r', 'r', 'o', 'r', '\x1b', '[', '0', 'm', '\n']) + def test_progress_nocolor_no_crash(self): + """progress() + clear_progress() must not crash when color=False. + + Regression: clear_progress() used to do an unconditional `import colorama` + and call colorama.ansi.clear_line(), which fails if colorama isn't installed. + """ + + with OutputIO() as buf: + with redirect_stderr(buf): + l = LogConsole(show_verbose=False, show_debug=False, color=False) + l.progress("working") + l.progress("still working") # triggers clear_progress on the previous line + l.clear_progress() + # write something visible after clearing + l.error("done") + + self.assertIn("done", buf.getvalue()) + def test_nocolor(self): """test without color output""" diff --git a/tests/test_thinner.py b/tests/test_thinner.py index 4425d3d..097ec32 100644 --- a/tests/test_thinner.py +++ b/tests/test_thinner.py @@ -99,6 +99,51 @@ class TestThinner(unittest2.TestCase): self.assertEqual(result, ok) + def test_always_keep_only(self): + """Thinner('N') with no rules: keep at most N most-recent objects, regardless of age.""" + + thinner = Thinner("3") + + # fewer than N -> keep them all, remove nothing + few = [Thing(100), Thing(200)] + (keeps, removes) = thinner.thin(few, keep_objects=[], now=1000) + self.assertEqual(keeps, few) + self.assertEqual(removes, []) + + # exactly N -> keep them all, remove nothing + exact = [Thing(100), Thing(200), Thing(300)] + (keeps, removes) = thinner.thin(exact, keep_objects=[], now=1000) + self.assertEqual(keeps, exact) + self.assertEqual(removes, []) + + # more than N -> keep the last N, drop the rest + many = [Thing(100), Thing(200), Thing(300), Thing(400), Thing(500)] + (keeps, removes) = thinner.thin(many, keep_objects=[], now=1000) + self.assertEqual(keeps, many[-3:]) + self.assertEqual(removes, many[:-3]) + + def test_keep_objects_without_timestamp(self): + """objects with timestamp=None are always kept (treated as 'not ours').""" + + thinner = Thinner("1d1w") + no_ts = Thing(None) + old_ts = Thing(100) # well past the ttl + + (keeps, removes) = thinner.thin([no_ts, old_ts], keep_objects=[], now=10 ** 9) + self.assertIn(no_ts, keeps) + self.assertIn(old_ts, removes) + + def test_empty_thinner_keeps_everything(self): + """Thinner('') has no rules and always_keep=0 -> nothing is kept by rules, but timestamp=None objects still pass through.""" + + thinner = Thinner("") + with_ts = Thing(100) + without_ts = Thing(None) + + (keeps, removes) = thinner.thin([with_ts, without_ts], keep_objects=[], now=1000) + self.assertEqual(keeps, [without_ts]) + self.assertEqual(removes, [with_ts]) + def test_full(self): ok=['2022-03-09 01:56:23', '2023-01-03 10:53:16', diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 0a54245..8eee85f 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -538,6 +538,40 @@ test_target1/test_source2/fs2/sub@snap-test-2010-11-11 self.assertEqual(cm.exception.code, 255) self.assertIn("Invalid tag seperator", buf.getvalue()) + def test_target_path_absolute_rejected(self): + """TARGET-PATH must be a ZFS path; a leading '/' is rejected (it's not a filesystem mountpoint).""" + + with OutputIO() as buf: + with redirect_stderr(buf): + with self.assertRaises(SystemExit) as cm: + ZfsAutobackup("test /test_target1 --no-progress".split(" ")) + self.assertEqual(cm.exception.code, 255) + self.assertIn("Target should not start with a /", buf.getvalue()) + + def test_tag_contains_seperator_rejected(self): + """--tag must not contain the tag-seperator (otherwise tagless_suffix parsing breaks).""" + + with OutputIO() as buf: + with redirect_stderr(buf): + with self.assertRaises(SystemExit) as cm: + # default tag-seperator is "__"; a tag containing "__" should be rejected. + ZfsAutobackup("test test_target1 --no-progress --tag=foo__bar".split(" ")) + self.assertEqual(cm.exception.code, 255) + self.assertIn("may not contain tag seperator", buf.getvalue()) + + def test_ignore_replicated_renamed_warning(self): + """--ignore-replicated is the old name; it should warn and be mapped to --exclude-unchanged.""" + + with mocktime("20101111000000"): + with OutputIO() as buf: + with redirect_stdout(buf): + # --ignore-replicated is hidden in --help but still accepted; run a no-op + # invocation so we can capture the rename warning without doing real work. + self.assertFalse(ZfsAutobackup( + "test --no-progress --verbose --no-snapshot --ignore-replicated".split(" ")).run()) + + self.assertIn("--ignore-replicated has been renamed", buf.getvalue()) + def test_hold_format(self): """--hold-format controls the zfs-hold name placed on snapshots.""" diff --git a/zfs_autobackup/CmdPipe.py b/zfs_autobackup/CmdPipe.py index 29d0e5c..6a93d79 100644 --- a/zfs_autobackup/CmdPipe.py +++ b/zfs_autobackup/CmdPipe.py @@ -144,20 +144,20 @@ class CmdPipe: for item in self.items: if item.process.stdout in read_ready: - line = item.process.stdout.readline().decode('utf-8').rstrip() - if line != "": - item.stdout_handler(line) - else: + raw = item.process.stdout.readline() + if raw == b"": eof_count = eof_count + 1 if item.next: item.next.process.stdin.close() + else: + item.stdout_handler(raw.decode('utf-8').rstrip()) if item.process.stderr in read_ready: - line = item.process.stderr.readline().decode('utf-8').rstrip() - if line != "": - item.stderr_handler(line) - else: + raw = item.process.stderr.readline() + if raw == b"": eof_count = eof_count + 1 + else: + item.stderr_handler(raw.decode('utf-8').rstrip()) if item.process.poll() is not None: done_count = done_count + 1 diff --git a/zfs_autobackup/ExecuteNode.py b/zfs_autobackup/ExecuteNode.py index 892729d..931e95e 100644 --- a/zfs_autobackup/ExecuteNode.py +++ b/zfs_autobackup/ExecuteNode.py @@ -258,7 +258,7 @@ class ExecuteNode(LogStub): cmd.append("ssh") if self.ssh_config is not None: - cmd.append(["-F", self.ssh_config]) + cmd.extend(["-F", self.ssh_config]) cmd.append(self.ssh_to) diff --git a/zfs_autobackup/LogConsole.py b/zfs_autobackup/LogConsole.py index ebec45e..fd1aa99 100644 --- a/zfs_autobackup/LogConsole.py +++ b/zfs_autobackup/LogConsole.py @@ -3,28 +3,26 @@ from __future__ import print_function import sys +try: + import colorama + _COLORAMA_AVAILABLE = True +except ImportError: + _COLORAMA_AVAILABLE = False + class LogConsole: """Log-class that outputs to console, adding colors if needed""" + # ANSI escape used as a fallback when colorama isn't available + _CLEAR_LINE = "\033[2K\r" + def __init__(self, show_debug, show_verbose, color): self.last_log = "" self.show_debug = show_debug self.show_verbose = show_verbose self._progress_uncleared = False - if color: - # try to use color, failback if colorama not available - self.colorama = False - try: - import colorama - global colorama - self.colorama = True - except ImportError: - pass - - else: - self.colorama = False + self.colorama = color and _COLORAMA_AVAILABLE def error(self, txt): self.clear_progress() @@ -69,7 +67,8 @@ class LogConsole: def clear_progress(self): if self._progress_uncleared: - import colorama - print(colorama.ansi.clear_line(), end='', file=sys.stderr) - # sys.stderr.flush() + if self.colorama: + print(colorama.ansi.clear_line(), end='', file=sys.stderr) + else: + print(self._CLEAR_LINE, end='', file=sys.stderr) self._progress_uncleared = False diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 0b4999f..f72c4ae 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -396,7 +396,7 @@ class ZfsAutobackup(ZfsAuto): # determine common zpool features (cached, so no problem we call it often) source_features = source_node.get_pool(source_dataset).features target_features = target_node.get_pool(target_dataset).features - common_features = source_features and target_features + common_features = [f for f in source_features if f in target_features] if self.args.no_bookmarks: use_bookmarks = False diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index 0b8242f..2fd45e7 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -290,11 +290,11 @@ class ZfsSnapshot(ZfsPointInTime): self.debug("Rolling back") self.zfs_node.run(["zfs", "rollback", self.name]) - def destroy(self, fail_exception=False, defered=False): + def destroy(self, fail_exception=False, deferred=False): self.verbose("Destroying") self.release() - return super().destroy(fail_exception=fail_exception, deferred=defered) + return super().destroy(fail_exception=fail_exception, deferred=deferred) def clone(self, name): """clones this snapshot and returns ZfsContainer of the clone From 32b2012e420e5f96c0599f14438b0779b1796443 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 11:04:19 +0200 Subject: [PATCH 090/135] fix test paths --- tests/run_test | 3 ++- tests/run_tests | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/run_test b/tests/run_test index 15ffdfc..c400fd1 100755 --- a/tests/run_test +++ b/tests/run_test @@ -2,4 +2,5 @@ #run all or one test. start from main directory # to run one: -k test_incremental_resume -python -m unittest discover tests $@ -vvvf +cd tests +python -m unittest discover .. $@ -vvvf diff --git a/tests/run_tests b/tests/run_tests index 3b5d6ec..ea7f6c0 100755 --- a/tests/run_tests +++ b/tests/run_tests @@ -31,7 +31,8 @@ modprobe brd rd_size=512000 umount /tmp/ZfsCheck* -coverage run --branch --source zfs_autobackup -m unittest discover -vvvvf $SCRIPTDIR $@ 2>&1 +cd $SCRIPTDIR +coverage run --branch --source zfs_autobackup -m unittest discover -vvvvf .. $@ 2>&1 EXIT=$? echo From 557b514519007615e25f35e5aba7f8d7063219c2 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 11:21:25 +0200 Subject: [PATCH 091/135] fix tests --- tests/test_zfscheck.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_zfscheck.py b/tests/test_zfscheck.py index b89eb09..a45fac1 100644 --- a/tests/test_zfscheck.py +++ b/tests/test_zfscheck.py @@ -194,7 +194,7 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bd8 # breaks pipe when head exists # important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling - shelltest("python -m zfs_autobackup.ZfsCheck test_source1@test --debug | head -n1") + shelltest("cd ..;python3 -m zfs_autobackup.ZfsCheck test_source1@test --debug | head -n1") # should NOT be mounted anymore if cleanup went ok: self.assertNotRegex(shelltest("mount"), "test_source1@test") @@ -209,7 +209,7 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bd8 # breaks pipe when grep exists: # important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling - shelltest("python3 -m zfs_autobackup.ZfsCheck test_source1/vol@test --debug| grep -m1 'Hashing file'") + shelltest("cd ..;python3 -m zfs_autobackup.ZfsCheck test_source1/vol@test --debug| grep -m1 'Hashing file'") # time.sleep(1) r = shelltest("zfs list -H -o name -r -t snapshot,filesystem,volume " + TEST_POOLS) From caf3b526a9acce21ff8176469e2e0ed45c8572dc Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 11:43:31 +0200 Subject: [PATCH 092/135] beta2 --- zfs_autobackup/CliBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zfs_autobackup/CliBase.py b/zfs_autobackup/CliBase.py index e3444c8..aced9d6 100644 --- a/zfs_autobackup/CliBase.py +++ b/zfs_autobackup/CliBase.py @@ -10,7 +10,7 @@ class CliBase(object): Overridden in subclasses that add stuff for the specific programs.""" # also used by setup.py - VERSION = "4.0-beta1" + VERSION = "4.0-beta2" HEADER = "{} v{} - (c)2026 E.H.Eefting (edwin@datux.nl)".format(os.path.basename(sys.argv[0]), VERSION) def __init__(self, argv, print_arguments=True): From a8b34d792cb6c05bea2a624f5e73b32168e0c1d7 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 12:43:04 +0200 Subject: [PATCH 093/135] show bookmark_tag --- zfs_autobackup/ZfsAutobackup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index f72c4ae..e8c65cf 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -550,6 +550,9 @@ class ZfsAutobackup(ZfsAuto): raise (Exception( "Target path '{}' does not exist. Please create this dataset first.".format(target_dataset))) + bookmark_tag = target_dataset.properties['guid'] + target_node.verbose("Bookmark tag: {}".format(bookmark_tag)) + # check for collisions due to strip-path self.check_target_names(source_node, source_datasets, target_node) @@ -559,7 +562,7 @@ class ZfsAutobackup(ZfsAuto): source_node=source_node, source_datasets=source_datasets, target_node=target_node, - bookmark_tag=target_dataset.properties['guid']) + bookmark_tag=bookmark_tag) # no target specified, run in snapshot-only mode else: From a789251b305c93003cc0b803e516733fbfdee8a0 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 13:29:48 +0200 Subject: [PATCH 094/135] fixes --- zfs_autobackup/ZfsAutobackup.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index e8c65cf..1ab98a9 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -31,6 +31,15 @@ class ZfsAutobackup(ZfsAuto): if args.allow_empty: args.min_change = 0 + # With --keep-source=0 nothing survives on the source between runs, so the + # per-dataset is_changed_ours() check has no reference snapshot to compare + # against — every dataset would get a fresh snapshot anyway. Force min-change=0 + # to skip the our_snapshots / written@... lookups entirely (significant speedup + # on large dataset counts). + if args.keep_source == "0" and args.min_change > 0: + self.verbose("NOTE: --keep-source=0: forcing --min-change=0 (no source reference to compare against).") + args.min_change = 0 + # if args.destroy_incompatible: # args.rollback = True @@ -542,8 +551,6 @@ class ZfsAutobackup(ZfsAuto): thinner=target_thinner, tag_seperator=self.tag_seperator) target_node.verbose("Receive datasets under: {}".format(self.args.target_path)) - self.set_title("Synchronising") - # check if exists, to prevent vague errors target_dataset = target_node.get_dataset(self.args.target_path) if not target_dataset.exists: @@ -553,6 +560,8 @@ class ZfsAutobackup(ZfsAuto): bookmark_tag = target_dataset.properties['guid'] target_node.verbose("Bookmark tag: {}".format(bookmark_tag)) + self.set_title("Synchronising") + # check for collisions due to strip-path self.check_target_names(source_node, source_datasets, target_node) From bdabb72a79dbe64eaa07123897952f11db8770a7 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 15:46:19 +0200 Subject: [PATCH 095/135] fixes --- zfs_autobackup/ZfsAutobackup.py | 21 +++++++++++---------- zfs_autobackup/ZfsContainer.py | 13 +++++++------ zfs_autobackup/ZfsNode.py | 10 +++++----- zfs_autobackup/ZfsSnapshot.py | 13 ++++--------- 4 files changed, 27 insertions(+), 30 deletions(-) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 1ab98a9..18f701a 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -1,15 +1,14 @@ import argparse from signal import signal, SIGPIPE -from .util import output_redir, sigpipe_handler, datetime_now - -from .ZfsAuto import ZfsAuto from . import compressors from .ExecuteNode import ExecuteNode from .Thinner import Thinner -from .ZfsDataset import ZfsDataset -from .ZfsNode import ZfsNode from .ThinnerRule import ThinnerRule +from .ZfsAuto import ZfsAuto +from .ZfsContainer import ZfsContainer +from .ZfsNode import ZfsNode +from .util import sigpipe_handler, datetime_now class ZfsAutobackup(ZfsAuto): @@ -161,8 +160,8 @@ class ZfsAutobackup(ZfsAuto): # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: def thin_missing_targets(self, target_dataset, used_target_datasets): """thin target datasets that are missing on the source. - :type used_target_datasets: list[ZfsDataset] - :type target_dataset: ZfsDataset + :type used_target_datasets: list[ZfsContainer] + :type target_dataset: ZfsContainer """ self.debug("Thinning obsolete datasets") @@ -190,8 +189,8 @@ class ZfsAutobackup(ZfsAuto): # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: def destroy_missing_targets(self, target_dataset, used_target_datasets): """destroy target datasets that are missing on the source and that meet the requirements - :type used_target_datasets: list[ZfsDataset] - :type target_dataset: ZfsDataset + :type used_target_datasets: list[ZfsContainer] + :type target_dataset: ZfsContainer """ @@ -370,7 +369,7 @@ class ZfsAutobackup(ZfsAuto): """Sync datasets, or thin-only on both sides :type bookmark_tag: str :type target_node: ZfsNode - :type source_datasets: list of ZfsDataset + :type source_datasets: list of ZfsContainer :type source_node: ZfsNode """ @@ -391,6 +390,7 @@ class ZfsAutobackup(ZfsAuto): # determine corresponding target_dataset target_name = self.make_target_name(source_dataset) target_dataset = target_node.get_dataset(target_name) + assert isinstance(target_dataset, ZfsContainer) target_datasets.append(target_dataset) # ensure parents exists @@ -440,6 +440,7 @@ class ZfsAutobackup(ZfsAuto): raise target_path_dataset = target_node.get_dataset(self.args.target_path) + assert isinstance(target_path_dataset, ZfsContainer) if not self.args.no_thinning: self.thin_missing_targets(target_dataset=target_path_dataset, used_target_datasets=target_datasets) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 17c00bb..62731ca 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -1,6 +1,7 @@ from .ZfsBookmark import ZfsBookmark from .ZfsDataset import ZfsDataset from .ExecuteNode import ExecuteError +from .ZfsPointInTime import ZfsPointInTime from .ZfsSnapshot import ZfsSnapshot @@ -173,16 +174,16 @@ class ZfsContainer(ZfsDataset): Args: :rtype: ZfsSnapshot|None - :type snapshot_name: str|ZfsSnapshot|None + :type snapshot_name: str|ZfsPointInTime|None """ if snapshot_name is None: return None - if not isinstance(snapshot_name, ZfsDataset): - tagless_suffix = snapshot_name - else: + if isinstance(snapshot_name, ZfsPointInTime): tagless_suffix = snapshot_name.tagless_suffix + else: + tagless_suffix = snapshot_name for snapshot_name in self.snapshots: if snapshot_name.tagless_suffix == tagless_suffix: @@ -389,7 +390,7 @@ class ZfsContainer(ZfsDataset): def find_bookmark(self, snapshot_bookmark, preferred_tag): """find bookmark by bookmark name (can be a suffix or a different - ZfsDataset) Returns None if it cant find it. + ZfsSnapshot or ZfsBookmark) Returns None if it cant find it. We try to find the bookmark with the preferred tag (which is usually a target path guid, to prevent conflicting bookmarks by multiple sends) If that fails, we return any bookmark that matches (and ignore the tag) @@ -431,7 +432,7 @@ class ZfsContainer(ZfsDataset): def find_snapshot_index(self, snapshot): """find exact snapshot index by snapshot (can be a snapshot_name or - ZfsDataset) + ZfsSnapshot) Args: :type snapshot: str or ZfsSnapshot diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index 2d451c8..e26b521 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -72,9 +72,9 @@ class ZfsNode(ExecuteNode): """Returns a list of snapshots to keep and remove, according to current time and thinner settings. :return: ( keeps, removes ) - :type snapshots: list[ZfsDataset] - :type keep_snapshots: list[ZfsDataset] - :rtype: ( list[ZfsDataset], list[ZfsDataset] ) + :type snapshots: list[ZfsSnapshot] + :type keep_snapshots: list[ZfsSnapshot] + :rtype: ( list[ZfsSnapshot], list[ZfsSnapshot] ) """ # NOTE: if thinning is disabled with --no-thinning, self.__thinner will be none. if self.__thinner is not None: @@ -289,7 +289,7 @@ class ZfsNode(ExecuteNode): def selected_datasets(self, property_name, exclude_received, exclude_paths, exclude_unchanged): """determine filesystems that should be backed up by looking at the special autobackup-property, systemwide - returns: ( list of selected ZfsDataset, list of excluded ZfsDataset) + returns: ( list of selected ZfsContainer, list of excluded ZfsContainer) """ self.debug("Getting selected datasets") @@ -300,7 +300,7 @@ class ZfsNode(ExecuteNode): property_name ]) - # The returnlist of selected ZfsDataset's: + # The returnlist of selected ZfsContainer's: selected_filesystems = [] excluded_filesystems = [] diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index 2fd45e7..245fc43 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -1,8 +1,3 @@ -from .ExecuteNode import ExecuteError -import re -import sys -import time - from .ZfsPointInTime import ZfsPointInTime @@ -58,7 +53,7 @@ class ZfsSnapshot(ZfsPointInTime): :type send_pipes: list[str] :type features: list[str] :type prev_snapshot: ZfsSnapshot|ZfsBookmark|None - :type resume_token: str + :type resume_token: str|None :type show_progress: bool :type raw: bool """ @@ -195,14 +190,14 @@ class ZfsSnapshot(ZfsPointInTime): Args: :type send_pipes: list[str] :type recv_pipes: list[str] - :type target_snapshot: ZfsDataset + :type target_snapshot: ZfsSnapshot :type features: list[str] - :type prev_snapshot: ZfsDataset + :type prev_snapshot: ZfsSnapshot|ZfsBookmark|None :type show_progress: bool :type filter_properties: list[str] :type set_properties: list[str] :type ignore_recv_exit_code: bool - :type resume_token: str + :type resume_token: str|None :type raw: bool """ From 9b28219986e26ba2846165b9a825848fd829ca08 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 16:00:42 +0200 Subject: [PATCH 096/135] typing --- zfs_autobackup/ZfsAutobackup.py | 9 +++++---- zfs_autobackup/ZfsContainer.py | 20 ++++++++++++++------ zfs_autobackup/ZfsPointInTime.py | 2 +- zfs_autobackup/ZfsSnapshot.py | 1 + 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 18f701a..d0708df 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -395,12 +395,13 @@ class ZfsAutobackup(ZfsAuto): # ensure parents exists # TODO: this isnt perfect yet, in some cases it can create parents when it shouldn't. + target_parent = target_dataset.parent if not self.args.no_send \ - and target_dataset.parent \ - and target_dataset.parent not in target_datasets \ - and not target_dataset.parent.exists: + and target_parent is not None \ + and target_parent not in target_datasets \ + and not target_parent.exists: target_dataset.debug("Creating unmountable parents") - target_dataset.parent.create_filesystem(parents=True) + target_parent.create_filesystem(parents=True) # determine common zpool features (cached, so no problem we call it often) source_features = source_node.get_pool(source_dataset).features diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 62731ca..a47d2a3 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -108,7 +108,9 @@ class ZfsContainer(ZfsDataset): if target_common_snapshot and self.snapshots: followup = True - for snapshot in self.snapshots[self.find_snapshot_index(target_common_snapshot) + 1:]: + common_index = self.find_snapshot_index(target_common_snapshot) + assert common_index is not None + for snapshot in self.snapshots[common_index + 1:]: if raw or not followup or int(snapshot.properties['written']) != 0: followup = False ret.append(snapshot) @@ -117,7 +119,9 @@ class ZfsContainer(ZfsDataset): @property def written_since_ours(self): - """get number of bytes written since our last snapshot""" + """get number of bytes written since our last snapshot + :rtype: int + """ if self.__written_since_ours is None: latest_snapshot = self.our_snapshots[-1] @@ -334,8 +338,9 @@ class ZfsContainer(ZfsDataset): """ # recurse up - if parents and self.parent and not self.parent.exists: - self.parent.create_filesystem(parents, unmountable) + parent = self.parent + if parents and parent is not None and not parent.exists: + parent.create_filesystem(parents, unmountable) cmd = ["zfs", "create"] @@ -690,6 +695,7 @@ class ZfsContainer(ZfsDataset): for target_snapshot in target_transfers: source_snapshot = self.find_snapshot(target_snapshot) + assert source_snapshot is not None # do the rollback, one time at first transfer if do_rollback: @@ -735,14 +741,14 @@ class ZfsContainer(ZfsDataset): source_snapshot.hold() # release hold, cleanup obsolete snapshot - if prev_source_snapshot_bookmark and type(prev_source_snapshot_bookmark) is ZfsSnapshot: + if isinstance(prev_source_snapshot_bookmark, ZfsSnapshot): prev_source_snapshot_bookmark.release() if prev_source_snapshot_bookmark in source_obsoletes: prev_source_snapshot_bookmark.destroy() # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, # the rest was already destroyed or will not be send) - if prev_target_snapshot in target_obsoletes: + if prev_target_snapshot is not None and prev_target_snapshot in target_obsoletes: prev_target_snapshot.destroy() # we always try to use the bookmark during incremental send @@ -879,3 +885,5 @@ class ZfsContainer(ZfsDataset): target_dataset.abort_resume() else: return resume_token + + return None diff --git a/zfs_autobackup/ZfsPointInTime.py b/zfs_autobackup/ZfsPointInTime.py index b30af10..8e738e5 100644 --- a/zfs_autobackup/ZfsPointInTime.py +++ b/zfs_autobackup/ZfsPointInTime.py @@ -34,7 +34,7 @@ class ZfsPointInTime(ZfsDataset): def parent(self): """get parent dataset - :rtype: ZfsContainer | None + :rtype: ZfsContainer """ return self.zfs_node.get_dataset(self.prefix) diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index 245fc43..b36c770 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -27,6 +27,7 @@ class ZfsSnapshot(ZfsPointInTime): if pattern.search(suffix) is not None: self.debug("Excluded (path matches snapshot exclude pattern)") return True + return False def find_snapshot_by_suffix(self, snapshot_bookmarks): """return ZfsSnapshot|ZfsBookmark from the list of snapshot_bookmarks, if it matches our snapshot_name. Otherwise None From a478a2078742e87b7bc728c6307f2c74e4d2f2ee Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 16:06:20 +0200 Subject: [PATCH 097/135] fixes --- zfs_autobackup/ZfsContainer.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index a47d2a3..3801e00 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -5,10 +5,7 @@ from .ZfsPointInTime import ZfsPointInTime from .ZfsSnapshot import ZfsSnapshot -# import re -# from datetime import datetime -# import sys -# import time + class ZfsContainer(ZfsDataset): @@ -172,7 +169,7 @@ class ZfsContainer(ZfsDataset): def find_snapshot(self, snapshot_name): """find snapshot by snapshot name (can be a suffix or a different - ZfsSnapshot) Returns None if it cant find it. + ZfsSnapshot) Returns None if it can't find it. Note that matches with our own snapshots will be done tagless. @@ -215,7 +212,7 @@ class ZfsContainer(ZfsDataset): def automount(self): """Mount the dataset as if one did a zfs mount -a, but only for this dataset - Failure to mount doesnt result in an exception, but outputs errors to STDERR. + Failure to mount doesn't result in an exception, but outputs errors to STDERR. """ @@ -335,6 +332,7 @@ class ZfsContainer(ZfsDataset): Args: :type parents: bool + :type unmountable: bool """ # recurse up @@ -356,6 +354,7 @@ class ZfsContainer(ZfsDataset): """Update our snapshot and bookmark cache (if we have any). Use force if you want to force the caching, potentially triggering a zfs list Args: :type snapshot: ZfsSnapshot|ZfsBookmark + :type force: bool """ if force: @@ -395,7 +394,7 @@ class ZfsContainer(ZfsDataset): def find_bookmark(self, snapshot_bookmark, preferred_tag): """find bookmark by bookmark name (can be a suffix or a different - ZfsSnapshot or ZfsBookmark) Returns None if it cant find it. + ZfsSnapshot or ZfsBookmark) Returns None if it can't find it. We try to find the bookmark with the preferred tag (which is usually a target path guid, to prevent conflicting bookmarks by multiple sends) If that fails, we return any bookmark that matches (and ignore the tag) @@ -629,8 +628,12 @@ class ZfsContainer(ZfsDataset): :type holds: bool :type rollback: bool :type decrypt: bool + :type encrypt: bool :type also_other_snapshots: bool :type no_send: bool + :type destroy_incompatible: bool + :type zfs_compressed: bool + :type force: bool :type guid_check: bool :type use_bookmarks: bool :type bookmark_tag: str @@ -647,7 +650,7 @@ class ZfsContainer(ZfsDataset): if self.properties.get('encryption', 'off') != 'off': # user wants to send it over decrypted? if decrypt: - # when decrypting, zfs cant send properties + # when decrypting, zfs can't send properties send_properties = False else: # keep data encrypted by sending it raw (including properties) @@ -678,9 +681,9 @@ class ZfsContainer(ZfsDataset): set_properties) # always filter properties that start with 'autobackup:' (https://github.com/psy0rz/zfs_autobackup/issues/221) - for property in self.properties: - if property.startswith('autobackup:'): - active_filter_properties.append(property) + for prop in self.properties: + if prop.startswith('autobackup:'): + active_filter_properties.append(prop) # encrypt at target? if encrypt and not raw: From f828755f147ab3fde46048a99546d367128c716f Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 16:10:42 +0200 Subject: [PATCH 098/135] fixes --- zfs_autobackup/ZfsDataset.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 5beffab..9e12868 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1,5 +1,10 @@ +from typing import TYPE_CHECKING, cast + from .ExecuteNode import ExecuteError +if TYPE_CHECKING: + from .ZfsNode import ZfsNode + # # Inheritance / overview graph (comment): # @@ -140,9 +145,10 @@ class ZfsDataset: if self.__exists_check is None: self.debug("Checking if dataset exists") - self.__exists_check = (len(self.zfs_node.run(tab_split=True, cmd=["zfs", "list", self.name], readonly=True, - valid_exitcodes=[0, 1], - hide_errors=True)) > 0) + output = cast("list[str]", self.zfs_node.run( + tab_split=True, cmd=["zfs", "list", self.name], readonly=True, + valid_exitcodes=[0, 1], hide_errors=True)) + self.__exists_check = (len(output) > 0) return self.__exists_check @@ -187,7 +193,9 @@ class ZfsDataset: @property def properties(self): - """all zfs properties""" + """all zfs properties + :rtype: dict[str, str] + """ if self.__properties is None: @@ -197,10 +205,13 @@ class ZfsDataset: self.debug("Getting zfs properties") - self.__properties = {} - for pair in self.zfs_node.run(tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]): + properties = {} # type: dict[str, str] + output = cast("list[list[str]]", cast(object, self.zfs_node.run( + tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]))) + for pair in output: if len(pair) == 2: - self.__properties[pair[0]] = pair[1] + properties[pair[0]] = pair[1] + self.__properties = properties return self.__properties From 29142986425e2d9528ad67186af72bf860691491 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 16:14:52 +0200 Subject: [PATCH 099/135] fixes --- zfs_autobackup/ZfsBookmark.py | 6 ------ zfs_autobackup/ZfsSnapshot.py | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/zfs_autobackup/ZfsBookmark.py b/zfs_autobackup/ZfsBookmark.py index c2d2cce..418bab0 100644 --- a/zfs_autobackup/ZfsBookmark.py +++ b/zfs_autobackup/ZfsBookmark.py @@ -1,9 +1,3 @@ -from .ExecuteNode import ExecuteError -import re -from datetime import datetime -import sys -import time - from .ZfsPointInTime import ZfsPointInTime diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index b36c770..9854954 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -1,3 +1,5 @@ +from typing import cast + from .ZfsPointInTime import ZfsPointInTime @@ -57,6 +59,9 @@ class ZfsSnapshot(ZfsPointInTime): :type resume_token: str|None :type show_progress: bool :type raw: bool + :type send_properties: bool + :type write_embedded: bool + :type zfs_compressed: bool """ # build source command cmd = [] @@ -119,6 +124,7 @@ class ZfsSnapshot(ZfsPointInTime): :type filter_properties: list[str] :type set_properties: list[str] :type ignore_exit_code: bool + :type force: bool """ if set_properties is None: @@ -200,6 +206,10 @@ class ZfsSnapshot(ZfsPointInTime): :type ignore_recv_exit_code: bool :type resume_token: str|None :type raw: bool + :type send_properties: bool + :type write_embedded: bool + :type zfs_compressed: bool + :type force: bool """ if set_properties is None: @@ -261,8 +271,8 @@ class ZfsSnapshot(ZfsPointInTime): @property def holds(self): - output = self.zfs_node.run(["zfs", "holds", "-H", self.name], valid_exitcodes=[0], tab_split=True, - readonly=True) + output = cast("list[list[str]]", cast(object, self.zfs_node.run( + ["zfs", "holds", "-H", self.name], valid_exitcodes=[0], tab_split=True, readonly=True))) return map(lambda fields: fields[1], output) def is_hold(self): From 850cd548ffbdcee27817311e0fb35e2eb96eeab6 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 17:05:51 +0200 Subject: [PATCH 100/135] fixes --- zfs_autobackup/ZfsContainer.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 3801e00..e63833b 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -1,3 +1,5 @@ +from typing import cast + from .ZfsBookmark import ZfsBookmark from .ZfsDataset import ZfsDataset from .ExecuteNode import ExecuteError @@ -84,8 +86,9 @@ class ZfsContainer(ZfsDataset): self.name ] - self.__snapshots_bookmarks = self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), - force_exists=True) + names = cast("list[str]", self.zfs_node.run(cmd=cmd, readonly=True)) + self.__snapshots_bookmarks = cast("list[ZfsSnapshot|ZfsBookmark]", + self.zfs_node.get_datasets(names, force_exists=True)) return self.__snapshots_bookmarks @@ -126,7 +129,7 @@ class ZfsContainer(ZfsDataset): self.debug("Getting bytes written since our last snapshot") cmd = ["zfs", "get", "-H", "-ovalue", "-p", "written@" + str(latest_snapshot), self.name] - output = self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0]) + output = cast("list[str]", self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0])) self.__written_since_ours = int(output[0]) @@ -298,11 +301,12 @@ class ZfsContainer(ZfsDataset): if self.__recursive_datasets is None: self.debug("Getting all recursive datasets under us") - names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ + names = cast("list[str]", self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name - ]) + ])) - self.__recursive_datasets = self.zfs_node.get_datasets(names[1:], force_exists=True) + self.__recursive_datasets = cast("list[ZfsContainer]", + self.zfs_node.get_datasets(names[1:], force_exists=True)) return self.__recursive_datasets @@ -319,11 +323,12 @@ class ZfsContainer(ZfsDataset): if self.__datasets is None: self.debug("Getting all datasets under us") - names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ + names = cast("list[str]", self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ "zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name - ]) + ])) - self.__datasets = self.zfs_node.get_datasets(names[1:], force_exists=True) + self.__datasets = cast("list[ZfsContainer]", + self.zfs_node.get_datasets(names[1:], force_exists=True)) return self.__datasets @@ -843,8 +848,8 @@ class ZfsContainer(ZfsDataset): # we want it? if (also_other_snapshots or source_snapshot.is_ours) and not source_snapshot.is_snapshot_excluded: # create virtual target snapshot - target_snapshot = target_dataset.zfs_node.get_dataset( - target_dataset.name + source_snapshot.typed_suffix, force_exists=False) + target_snapshot = cast(ZfsSnapshot, target_dataset.zfs_node.get_dataset( + target_dataset.name + source_snapshot.typed_suffix, force_exists=False)) possible_target_snapshots.append(target_snapshot) source_snapshot = self.find_next_snapshot(source_snapshot) From 49e81b8d0c21a25e6ff2a38ab62e2b3d23171e80 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 22:09:14 +0200 Subject: [PATCH 101/135] Much better handling of missing common snapshots. A missing common snapshot now is handled as all target-snapshots being incompatible. This means a missing common snapshot now can be solved with --destroy-incompatible -F. (it will tell when to do it since the options can be dangerous) --- tests/test_externalfailures.py | 5 +- zfs_autobackup/ZfsContainer.py | 158 +++++++++++++++++++++------------ 2 files changed, 102 insertions(+), 61 deletions(-) diff --git a/tests/test_externalfailures.py b/tests/test_externalfailures.py index 742818d..0347154 100644 --- a/tests/test_externalfailures.py +++ b/tests/test_externalfailures.py @@ -119,8 +119,6 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # remove corresponding source snapshot, so it becomes invalid shelltest("zfs destroy test_source1/fs1@test-20101111000000") - # NOTE: it can only abort the initial dataset if it has no subs - shelltest("zfs destroy test_target1/test_source1/fs1/sub; true") # --test try again, should abort old resume with mocktime("20101111000001"): @@ -159,7 +157,8 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # --test try again, should abort old resume with mocktime("20101111000002"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test --debug".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test".split(" ")).run()) + # try again, should abort old resume with mocktime("20101111000002"): diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index e63833b..5290efc 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -106,7 +106,12 @@ class ZfsContainer(ZfsDataset): ret = [] - if target_common_snapshot and self.snapshots: + if self.snapshots: + if target_common_snapshot is None: + # No common snapshot found: every existing target snapshot is in + # the way and must be handled by handle_incompatible_snapshots. + return list(self.snapshots) + followup = True common_index = self.find_snapshot_index(target_common_snapshot) assert common_index is not None @@ -242,6 +247,10 @@ class ZfsContainer(ZfsDataset): """abort current resume state""" self.debug("Aborting resume") self.zfs_node.run(["zfs", "recv", "-A", self.name]) + if not self.snapshots: + #abort of an initial resume + self.force_exists=False + def rollback(self): """rollback to latest existing snapshot on this dataset""" @@ -475,41 +484,37 @@ class ZfsContainer(ZfsDataset): :type bookmark_tag: str """ - if not target_dataset.exists or not target_dataset.snapshots: - # target has nothing yet - return None, None - else: - for target_snapshot in reversed(target_dataset.snapshots): + for target_snapshot in reversed(target_dataset.snapshots): - # Prefer bookmarks to snapshots - source_bookmark = self.find_bookmark(target_snapshot, preferred_tag=bookmark_tag) - if source_bookmark: - if guid_check and source_bookmark.properties['guid'] != target_snapshot.properties['guid']: - source_bookmark.warning("Bookmark has mismatching GUID, ignoring.") - else: - source_bookmark.debug("Common bookmark") - return source_bookmark, target_snapshot + # Prefer bookmarks to snapshots + source_bookmark = self.find_bookmark(target_snapshot, preferred_tag=bookmark_tag) + if source_bookmark: + if guid_check and source_bookmark.properties['guid'] != target_snapshot.properties['guid']: + source_bookmark.warning("Bookmark has mismatching GUID, ignoring.") + else: + source_bookmark.debug("Common bookmark") + return source_bookmark, target_snapshot - # Source snapshot with same suffix? - source_snapshot = self.find_snapshot(target_snapshot) - if source_snapshot: - if guid_check and source_snapshot.properties['guid'] != target_snapshot.properties['guid']: - source_snapshot.warning("Snapshot has mismatching GUID, ignoring.") - else: - source_snapshot.debug("Common snapshot") - return source_snapshot, target_snapshot + # Source snapshot with same suffix? + source_snapshot = self.find_snapshot(target_snapshot) + if source_snapshot: + if guid_check and source_snapshot.properties['guid'] != target_snapshot.properties['guid']: + source_snapshot.warning("Snapshot has mismatching GUID, ignoring.") + else: + source_snapshot.debug("Common snapshot") + return source_snapshot, target_snapshot - # Extensive GUID search (slower but works with all names) - try: - source_bookmark_snapshot = self.find_guid_bookmark_snapshot(target_snapshot.properties['guid']) - if source_bookmark_snapshot is not None: - return source_bookmark_snapshot, target_snapshot - except ExecuteError as e: - # in readonly mode we igore a failed property read for non existing snapshots - if not self.zfs_node.readonly: - raise e + # Extensive GUID search (slower but works with all names) + try: + source_bookmark_snapshot = self.find_guid_bookmark_snapshot(target_snapshot.properties['guid']) + if source_bookmark_snapshot is not None: + return source_bookmark_snapshot, target_snapshot + except ExecuteError as e: + # in readonly mode we igore a failed property read for non existing snapshots + if not self.zfs_node.readonly: + raise e - raise (Exception("Cant find common bookmark or snapshot with target. (Delete snapshots on target and use -F to fix this.)")) + return None, None def _pre_clean(self, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): """cleanup old stuff on the source before starting snapshot syncing @@ -672,15 +677,19 @@ class ZfsContainer(ZfsDataset): source_common_snapshot=source_common_snapshot, target_dataset=target_dataset, target_transfers=target_transfers, target_obsoletes=target_obsoletes, source_obsoletes=source_obsoletes) + # check if we can resume + if len(target_transfers)>0: + resume_token = self._validate_resume_token(target_dataset, target_transfers[0]) + else: + resume_token = None + # handle incompatible stuff on target - target_dataset.handle_incompatible_snapshots(incompatible_target_snapshots, destroy_incompatible) + target_dataset.handle_incompatible_target(incompatible_target_snapshots, destroy_incompatible, force, source_common_snapshot, resume_token is not None) # now actually transfer the snapshots, if we want if no_send or len(target_transfers) == 0: return - # check if we can resume - resume_token = self._validate_resume_token(target_dataset, target_transfers[0]) (active_filter_properties, active_set_properties) = self.get_allowed_properties(filter_properties, set_properties) @@ -767,29 +776,60 @@ class ZfsContainer(ZfsDataset): prev_target_snapshot = target_snapshot - def handle_incompatible_snapshots(self, incompatible_target_snapshots, destroy_incompatible): + def handle_incompatible_target(self, incompatible_target_snapshots, destroy_incompatible, force, source_common_snapshot, is_resume): """destroy incompatbile snapshots on target before sync, or inform user what to do Args: :type incompatible_target_snapshots: list[ZfsSnapshot] :type destroy_incompatible: bool + :type force: bool + :type source_common_snapshot: ZfsPointInTime|None + :type is_resume: bool + """ - if incompatible_target_snapshots: - if not destroy_incompatible: - for snapshot in incompatible_target_snapshots: - snapshot.error("Incompatible snapshot") - raise (Exception("Please destroy incompatible snapshots on target, or use --destroy-incompatible.")) - else: - for snapshot in incompatible_target_snapshots: - snapshot.verbose("Incompatible snapshot") - snapshot.destroy(fail_exception=True) - ### XXX: hacky, maybe not do it and have things check snapshot.exists instead? - self.snapshots.remove(snapshot) + if not self.exists: + # no target yet, so everything ok + return - if len(incompatible_target_snapshots) > 0: - self.rollback() + if not incompatible_target_snapshots and source_common_snapshot is not None: + # nice existing target with compatible common snapshot. + return + + if not incompatible_target_snapshots and is_resume and len(self.snapshots)==0: + # its a dataset from an incremental send we probably can resume, so its fine + return + + #from this point on things get progressively worse.. + + if source_common_snapshot: + # we just need to delete some incompatibles: + for snapshot in incompatible_target_snapshots: + snapshot.warning("Incompatible snapshot") + + if not destroy_incompatible: + raise (Exception("Use --destroy-incompatible to get rid of these.")) + + else: + # no common snapshot, so the whole dataset is incompatbile + if incompatible_target_snapshots: + if not force or not destroy_incompatible: + self.error("Incompatible dataset!") + raise (Exception("Use --destroy-incompatible -F to overwrite the target dataset and start over.")) + else: + if not force: + self.error("Incompatible dataset!") + raise (Exception("Use -F to overwrite the target dataset and start over.")) + + self.warning("Overwriting incompatible dataset.") + + # remove incompatibles and rollback + for snapshot in incompatible_target_snapshots: + snapshot.destroy(fail_exception=True) + + self.invalidate_cache() + self.rollback() def _plan_sync(self, target_dataset, also_other_snapshots, guid_check, raw, bookmark_tag): """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. @@ -811,15 +851,17 @@ class ZfsContainer(ZfsDataset): - list[ZfsSnapshot]: Incompatible target snapshots. Target snapshots that are in the way, after the common snapshot. (need to be destroyed to continue) """ - ### 1: determine common and start snapshot + ### 1: determine common and start snapshot if target already exists: - target_dataset.debug("Determining start snapshot") - (source_common_snapshot, target_common_snapshot) = self.find_common_snapshot(target_dataset, - guid_check=guid_check, - bookmark_tag=bookmark_tag) - # if source_common_snapshot: - # source_common_snapshot.verbose("Common snapshot or bookmark") - incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(target_common_snapshot, raw) + if target_dataset.exists: + target_dataset.debug("Determining start snapshot") + (source_common_snapshot, target_common_snapshot) = self.find_common_snapshot(target_dataset, + guid_check=guid_check, + bookmark_tag=bookmark_tag) + incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(target_common_snapshot, raw) + else: + source_common_snapshot=None + incompatible_target_snapshots =[] # let thinner decide whats obsolete on source after the transfer is done source_obsoletes = [] @@ -836,7 +878,7 @@ class ZfsContainer(ZfsDataset): possible_target_snapshots = [] # add all snapshots from the source, starting after the common snapshot if it exists - if source_common_snapshot: + if source_common_snapshot is not None: source_snapshot = self.find_next_snapshot(source_common_snapshot) else: if self.snapshots: From debd2d891b4a5fe4f4e09f01958ad64809b2f244 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 22:11:33 +0200 Subject: [PATCH 102/135] test for new missing common snapshot handling --- tests/test_zfsautobackup40.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py index 791d8b3..34f1cbc 100644 --- a/tests/test_zfsautobackup40.py +++ b/tests/test_zfsautobackup40.py @@ -284,6 +284,38 @@ test_target1/b/test_source2/fs2/sub@test-20101111000003 with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + def test_no_common_falls_back_to_incompatible(self): + """If no common snapshot/bookmark exists with target, target snapshots + must be reported as incompatible (instead of raising), so the user can + recover with --destroy-incompatible.""" + + # initial backup populates target and creates source bookmarks + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + # wipe every source snapshot and bookmark so nothing matches the target anymore + shelltest("zfs list -H -o name -t snapshot,bookmark -r test_source1 test_source2 " + "| xargs -rn1 zfs destroy") + + # without --destroy-incompatible the run must fail: target snapshots are in the way + with mocktime("20101111000001"): + self.assertTrue(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + # target snapshots from the initial backup must still be there (nothing destroyed yet) + r = shelltest("zfs list -H -o name -t snapshot test_target1/test_source1/fs1").strip() + self.assertIn("test-20101111000000", r) + + # with --destroy-incompatible -F the run succeeds: target is wiped and re-seeded + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --destroy-incompatible -F".split(" ")).run()) + + r = shelltest("zfs list -H -o name -t snapshot -r test_target1") + self.assertNotIn("test-20101111000000", r) + self.assertIn("test-20101111000002", r) + def test_keepsource0target10queuedsend_bookmarks(self): """Test if thinner doesnt destroy too much early on if there are no common snapshots YET. Issue #84""" # new behavior, with bookmarks. (will delete common snapshot, since there is a bookmark) From 3ab852b9b5d89047618abbc8ff775528fb2f4d14 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sat, 6 Jun 2026 22:41:22 +0200 Subject: [PATCH 103/135] fix --- zfs_autobackup/ZfsAutobackup.py | 3 ++- zfs_autobackup/ZfsContainer.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index d0708df..0552a81 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -431,7 +431,8 @@ class ZfsAutobackup(ZfsAuto): decrypt=self.args.decrypt, encrypt=self.args.encrypt, zfs_compressed=self.args.zfs_compressed, force=self.args.force, guid_check=not self.args.no_guid_check, use_bookmarks=use_bookmarks, - bookmark_tag=bookmark_tag) + bookmark_tag=bookmark_tag, + property_format=self.args.property_format) except Exception as e: fail_count = fail_count + 1 diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 5290efc..4aaf754 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -622,7 +622,7 @@ class ZfsContainer(ZfsDataset): def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, - use_bookmarks, bookmark_tag): + use_bookmarks, bookmark_tag, property_format): """sync this dataset's snapshots to target_dataset, while also thinning out old snapshots along the way. @@ -647,6 +647,7 @@ class ZfsContainer(ZfsDataset): :type guid_check: bool :type use_bookmarks: bool :type bookmark_tag: str + :type property_format: str """ # self.verbose("-> {}".format(target_dataset)) @@ -694,9 +695,10 @@ class ZfsContainer(ZfsDataset): (active_filter_properties, active_set_properties) = self.get_allowed_properties(filter_properties, set_properties) - # always filter properties that start with 'autobackup:' (https://github.com/psy0rz/zfs_autobackup/issues/221) + # always filter properties that start with the property-format prefix (https://github.com/psy0rz/zfs_autobackup/issues/221) + property_prefix = property_format.split(':')[0] for prop in self.properties: - if prop.startswith('autobackup:'): + if prop.startswith(property_prefix): active_filter_properties.append(prop) # encrypt at target? From 6b4563618c8bd45b1fb0e664b78651de140c90f4 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sun, 7 Jun 2026 11:20:36 +0200 Subject: [PATCH 104/135] fix --test option for incompatible snapshots --- tests/test_zfsautobackup40.py | 88 ++++++++++++++++++++++++++++++++ zfs_autobackup/ZfsContainer.py | 11 ++-- zfs_autobackup/ZfsDataset.py | 15 +++--- zfs_autobackup/ZfsNode.py | 5 ++ zfs_autobackup/ZfsPointInTime.py | 6 +++ 5 files changed, 110 insertions(+), 15 deletions(-) diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py index 34f1cbc..eab2b7b 100644 --- a/tests/test_zfsautobackup40.py +++ b/tests/test_zfsautobackup40.py @@ -316,6 +316,94 @@ test_target1/b/test_source2/fs2/sub@test-20101111000003 self.assertNotIn("test-20101111000000", r) self.assertIn("test-20101111000002", r) + def test_incompatible_dataset_test_mode(self): + """In --test mode, when no common snapshot exists, target dataset is + flagged as incompatible. Verify the output strings and that nothing + is actually changed.""" + + # initial backup + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + # wipe every source snapshot and bookmark so nothing matches target + shelltest("zfs list -H -o name -t snapshot,bookmark -r test_source1 test_source2 " + "| xargs -rn1 zfs destroy") + + # snapshot listing BEFORE running --test (to verify nothing changes) + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + # --test without --destroy-incompatible -F: must fail and report "Incompatible dataset!" + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000001"): + self.assertTrue(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertIn("Incompatible dataset", combined) + self.assertNotIn("STDERR", combined) + + + # --test WITH --destroy-incompatible -F: must succeed and warn "Overwriting incompatible dataset" + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --test --destroy-incompatible -F".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + self.assertIn("Overwriting incompatible dataset", combined) + self.assertNotIn("STDERR", combined) + + # --test must not have changed anything on disk + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + + def test_incompatible_snapshot_test_mode(self): + """In --test mode, when a common snapshot exists but extra incompatible + target snapshots are in the way, those must be reported as incompatible. + Verify the output string and that nothing is actually changed.""" + + # initial backup + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose".split(" ")).run()) + + # add an incompatible target snapshot (data written after common snapshot) + shelltest("touch /test_target1/test_source1/fs1/change.txt") + shelltest("zfs snapshot test_target1/test_source1/fs1@incompatible1") + + # snapshot listing BEFORE running --test (to verify nothing changes) + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + # --test without --destroy-incompatible: must fail and warn about "Incompatible snapshot" + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000001"): + self.assertTrue(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + self.assertIn("Incompatible snapshot", combined) + self.assertNotIn("STDERR", combined) + + # --test WITH --destroy-incompatible: must succeed and still report "Incompatible snapshot" + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --test --destroy-incompatible".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + self.assertIn("Incompatible snapshot", combined) + self.assertNotIn("STDERR", combined) + + # --test must not have changed anything on disk + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + def test_keepsource0target10queuedsend_bookmarks(self): """Test if thinner doesnt destroy too much early on if there are no common snapshots YET. Issue #84""" # new behavior, with bookmarks. (will delete common snapshot, since there is a bookmark) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 4aaf754..fd4715c 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -505,14 +505,9 @@ class ZfsContainer(ZfsDataset): return source_snapshot, target_snapshot # Extensive GUID search (slower but works with all names) - try: - source_bookmark_snapshot = self.find_guid_bookmark_snapshot(target_snapshot.properties['guid']) - if source_bookmark_snapshot is not None: - return source_bookmark_snapshot, target_snapshot - except ExecuteError as e: - # in readonly mode we igore a failed property read for non existing snapshots - if not self.zfs_node.readonly: - raise e + source_bookmark_snapshot = self.find_guid_bookmark_snapshot(target_snapshot.properties['guid']) + if source_bookmark_snapshot is not None: + return source_bookmark_snapshot, target_snapshot return None, None diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 9e12868..81172f9 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -64,13 +64,13 @@ class ZfsDataset: # caching self.__exists_check = None # type: None|bool - self.__properties = None # type: None|dict[str,str] + self._properties = None # type: None|dict[str,str] def invalidate_cache(self): """clear caches""" self.force_exists = None self.__exists_check = None - self.__properties = None + self._properties = None def __repr__(self): return "{}: {}".format(self.zfs_node, self.name) @@ -197,7 +197,7 @@ class ZfsDataset: :rtype: dict[str, str] """ - if self.__properties is None: + if self._properties is None: cmd = [ "zfs", "get", "-H", "-o", "property,value", "-p", "all", self.name @@ -211,9 +211,10 @@ class ZfsDataset: for pair in output: if len(pair) == 2: properties[pair[0]] = pair[1] - self.__properties = properties + self._properties = properties + + return self._properties - return self.__properties def get_allowed_properties(self, filter_properties, set_properties): """only returns lists of allowed properties for this dataset type @@ -249,7 +250,7 @@ class ZfsDataset: self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) # invalidate cache - self.__properties = None + self._properties = None def inherit(self, prop): """inherit zfs property""" @@ -263,7 +264,7 @@ class ZfsDataset: self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) # invalidate cache - self.__properties = None + self._properties = None def mount(self, mount_point): """Mount the container or snapshot at mount_point, if it is a filesystem.""" diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index e26b521..51cef7d 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -249,6 +249,11 @@ class ZfsNode(ExecuteNode): # force_exist, since we're making it snapshot = self.get_dataset(dataset.name + "@" + snapshot_name, force_exists=True) + assert(isinstance(snapshot,ZfsSnapshot)) + + if self.readonly: + snapshot.simulate_properties() + pool = dataset.split_path()[0] if pool not in pools: diff --git a/zfs_autobackup/ZfsPointInTime.py b/zfs_autobackup/ZfsPointInTime.py index 8e738e5..52333cf 100644 --- a/zfs_autobackup/ZfsPointInTime.py +++ b/zfs_autobackup/ZfsPointInTime.py @@ -100,3 +100,9 @@ class ZfsPointInTime(ZfsDataset): else: return None + + def simulate_properties(self): + """in test mode when we create fake snapshots or bookmarks, we also need to simluate properties""" + self._properties={ + 'guid': self.tagless_suffix # just use the snapshot name as "guid" + } From 34f27018de94c651c924064891098ffae44519d3 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sun, 7 Jun 2026 11:27:55 +0200 Subject: [PATCH 105/135] fix python name mangling headaces ( renamed __ to _ ) --- zfs_autobackup/ZfsContainer.py | 50 +++++++++++++++++----------------- zfs_autobackup/ZfsDataset.py | 10 +++---- zfs_autobackup/ZfsNode.py | 44 +++++++++++++++--------------- zfs_autobackup/ZfsPool.py | 12 ++++---- 4 files changed, 58 insertions(+), 58 deletions(-) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index fd4715c..d31c65d 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -17,10 +17,10 @@ class ZfsContainer(ZfsDataset): super().__init__(zfs_node, name, force_exists=force_exists) - self.__written_since_ours = None # type: None|int - self.__recursive_datasets = None # type: None|list[ZfsContainer] - self.__datasets = None # type: None|list[ZfsContainer] - self.__snapshots_bookmarks = None # type: None|list[ZfsSnapshot|ZfsBookmark] + self._written_since_ours = None # type: None|int + self._recursive_datasets = None # type: None|list[ZfsContainer] + self._datasets = None # type: None|list[ZfsContainer] + self._snapshots_bookmarks = None # type: None|list[ZfsSnapshot|ZfsBookmark] @property def parent(self): @@ -36,10 +36,10 @@ class ZfsContainer(ZfsDataset): def invalidate_cache(self): super().invalidate_cache() - self.__written_since_ours = None - self.__recursive_datasets = None - self.__datasets = None - self.__snapshots_bookmarks = None + self._written_since_ours = None + self._recursive_datasets = None + self._datasets = None + self._snapshots_bookmarks = None @property def snapshots(self): @@ -78,7 +78,7 @@ class ZfsContainer(ZfsDataset): """ # cached? - if self.__snapshots_bookmarks is None: + if self._snapshots_bookmarks is None: self.debug("Getting snapshots and bookmarks") cmd = [ @@ -87,10 +87,10 @@ class ZfsContainer(ZfsDataset): ] names = cast("list[str]", self.zfs_node.run(cmd=cmd, readonly=True)) - self.__snapshots_bookmarks = cast("list[ZfsSnapshot|ZfsBookmark]", - self.zfs_node.get_datasets(names, force_exists=True)) + self._snapshots_bookmarks = cast("list[ZfsSnapshot|ZfsBookmark]", + self.zfs_node.get_datasets(names, force_exists=True)) - return self.__snapshots_bookmarks + return self._snapshots_bookmarks def find_incompatible_snapshots(self, target_common_snapshot, raw): """returns a list[snapshots] that is incompatible for a zfs recv onto @@ -128,7 +128,7 @@ class ZfsContainer(ZfsDataset): :rtype: int """ - if self.__written_since_ours is None: + if self._written_since_ours is None: latest_snapshot = self.our_snapshots[-1] self.debug("Getting bytes written since our last snapshot") @@ -136,9 +136,9 @@ class ZfsContainer(ZfsDataset): output = cast("list[str]", self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0])) - self.__written_since_ours = int(output[0]) + self._written_since_ours = int(output[0]) - return self.__written_since_ours + return self._written_since_ours def is_changed(self, min_changed_bytes=1): """dataset is changed since ANY latest snapshot ? @@ -307,17 +307,17 @@ class ZfsContainer(ZfsDataset): :rtype: list[ZfsContainer] """ - if self.__recursive_datasets is None: + if self._recursive_datasets is None: self.debug("Getting all recursive datasets under us") names = cast("list[str]", self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name ])) - self.__recursive_datasets = cast("list[ZfsContainer]", - self.zfs_node.get_datasets(names[1:], force_exists=True)) + self._recursive_datasets = cast("list[ZfsContainer]", + self.zfs_node.get_datasets(names[1:], force_exists=True)) - return self.__recursive_datasets + return self._recursive_datasets @property def datasets(self, types="filesystem,volume"): @@ -329,17 +329,17 @@ class ZfsContainer(ZfsDataset): """ - if self.__datasets is None: + if self._datasets is None: self.debug("Getting all datasets under us") names = cast("list[str]", self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ "zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name ])) - self.__datasets = cast("list[ZfsContainer]", - self.zfs_node.get_datasets(names[1:], force_exists=True)) + self._datasets = cast("list[ZfsContainer]", + self.zfs_node.get_datasets(names[1:], force_exists=True)) - return self.__datasets + return self._datasets def create_filesystem(self, parents=False, unmountable=True): """create this container as a filesystem @@ -374,8 +374,8 @@ class ZfsContainer(ZfsDataset): if force: self.snapshots_bookmarks.append(snapshot) - elif self.__snapshots_bookmarks is not None: - self.__snapshots_bookmarks.append(snapshot) + elif self._snapshots_bookmarks is not None: + self._snapshots_bookmarks.append(snapshot) @property def our_snapshots(self): diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 81172f9..3ca09d0 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -63,13 +63,13 @@ class ZfsDataset: self.force_exists = force_exists # caching - self.__exists_check = None # type: None|bool + self._exists_check = None # type: None|bool self._properties = None # type: None|dict[str,str] def invalidate_cache(self): """clear caches""" self.force_exists = None - self.__exists_check = None + self._exists_check = None self._properties = None def __repr__(self): @@ -143,14 +143,14 @@ class ZfsDataset: def exists_check(self): """check on disk if it exists""" - if self.__exists_check is None: + if self._exists_check is None: self.debug("Checking if dataset exists") output = cast("list[str]", self.zfs_node.run( tab_split=True, cmd=["zfs", "list", self.name], readonly=True, valid_exitcodes=[0, 1], hide_errors=True)) - self.__exists_check = (len(output) > 0) + self._exists_check = (len(output) > 0) - return self.__exists_check + return self._exists_check @property def exists(self): diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index 51cef7d..5c65843 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -36,8 +36,8 @@ class ZfsNode(ExecuteNode): self.logger = logger - self.__supported_send_options = None - self.__supported_recv_options = None + self._supported_send_options = None + self._supported_recv_options = None self.exclude_snapshot_patterns = exclude_snapshot_patterns @@ -57,11 +57,11 @@ class ZfsNode(ExecuteNode): else: self.verbose("Keep no old snaphots") - self.__thinner = thinner + self._thinner = thinner # list of ZfsPools - self.__pools = {} # type: dict[str, ZfsPool] - self.__datasets = {} # type: dict[str, ZfsContainer | ZfsBookmark | ZfsSnapshot] + self._pools = {} # type: dict[str, ZfsPool] + self._datasets = {} # type: dict[str, ZfsContainer | ZfsBookmark | ZfsSnapshot] self._progress_total_bytes = 0 self._progress_start_time = time.time() @@ -77,9 +77,9 @@ class ZfsNode(ExecuteNode): :rtype: ( list[ZfsSnapshot], list[ZfsSnapshot] ) """ # NOTE: if thinning is disabled with --no-thinning, self.__thinner will be none. - if self.__thinner is not None: + if self._thinner is not None: - return self.__thinner.thin(snapshots, keep_snapshots, datetime_now(self.utc).timestamp()) + return self._thinner.thin(snapshots, keep_snapshots, datetime_now(self.utc).timestamp()) else: return (keep_snapshots, []) @@ -88,25 +88,25 @@ class ZfsNode(ExecuteNode): """list of supported options, for optimizing sends""" # not every zfs implementation supports them all - if self.__supported_send_options is None: - self.__supported_send_options = [] + if self._supported_send_options is None: + self._supported_send_options = [] for option in ["-L", "-e", "-c"]: if self.valid_command(["zfs", "send", option, "zfs_autobackup_option_test"]): - self.__supported_send_options.append(option) - return self.__supported_send_options + self._supported_send_options.append(option) + return self._supported_send_options @property def supported_recv_options(self): """list of supported options""" # not every zfs implementation supports them all - if self.__supported_recv_options is None: - self.__supported_recv_options = [] + if self._supported_recv_options is None: + self._supported_recv_options = [] for option in ["-s"]: if self.valid_command(["zfs", "recv", option, "zfs_autobackup_option_test"]): - self.__supported_recv_options.append(option) + self._supported_recv_options.append(option) - return self.__supported_recv_options + return self._supported_recv_options def valid_command(self, cmd): """test if a specified zfs options are valid exit code. use this to determine support options""" @@ -126,7 +126,7 @@ class ZfsNode(ExecuteNode): zpool_name = dataset.name.split("/")[0] - return self.__pools.setdefault(zpool_name, ZfsPool(self, zpool_name)) + return self._pools.setdefault(zpool_name, ZfsPool(self, zpool_name)) def get_dataset(self, name, force_exists=None): """get a ZfsDataset() object from name. stores objects internally to enable caching @@ -134,20 +134,20 @@ class ZfsNode(ExecuteNode): :rtype: ZfsContainer | ZfsBookmark | ZfsSnapshot """ - if name in self.__datasets: - return self.__datasets[name] + if name in self._datasets: + return self._datasets[name] if '@' in name: - self.__datasets[name] = ZfsSnapshot(self, name, force_exists=force_exists) + self._datasets[name] = ZfsSnapshot(self, name, force_exists=force_exists) elif '#' in name: - self.__datasets[name] = ZfsBookmark(self, name, force_exists=force_exists) + self._datasets[name] = ZfsBookmark(self, name, force_exists=force_exists) else: - self.__datasets[name] = ZfsContainer(self, name, force_exists=force_exists) + self._datasets[name] = ZfsContainer(self, name, force_exists=force_exists) - return self.__datasets[name] + return self._datasets[name] def get_datasets(self, names, force_exists=None): """get a list of ZfsDataset() object from names. stores objects internally to enable caching diff --git a/zfs_autobackup/ZfsPool.py b/zfs_autobackup/ZfsPool.py index 709fb7e..78bf5b8 100644 --- a/zfs_autobackup/ZfsPool.py +++ b/zfs_autobackup/ZfsPool.py @@ -10,10 +10,10 @@ class ZfsPool(): self.zfs_node = zfs_node self.name = name - self.__properties = None + self._properties = None def invalidate_cache(self): - self.__properties = None + self._properties = None def __repr__(self): return "{}: {}".format(self.zfs_node, self.name) @@ -40,7 +40,7 @@ class ZfsPool(): def properties(self): """all zpool properties""" - if self.__properties is None: + if self._properties is None: self.debug("Getting zpool properties") @@ -48,12 +48,12 @@ class ZfsPool(): "zpool", "get", "-H", "-p", "all", self.name ] - self.__properties = {} + self._properties = {} for pair in self.zfs_node.run(tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]): - self.__properties[pair[1]] = pair[2] + self._properties[pair[1]] = pair[2] - return self.__properties + return self._properties @property def features(self): From bd4a2c22ee5adbf121c74c695749866ce46babcc Mon Sep 17 00:00:00 2001 From: DatuX Date: Sun, 7 Jun 2026 12:13:56 +0200 Subject: [PATCH 106/135] --exclude-received no longer needed, using it results in error message with explanation. --- tests/test_encryption.py | 40 +++++++++++++++++------------------ tests/test_sendrecvpipes.py | 20 +++++++++--------- tests/test_verify.py | 18 ++++++++-------- tests/test_zfsautobackup.py | 4 ++-- tests/test_zfsautobackup40.py | 5 ----- zfs_autobackup/CliBase.py | 2 +- zfs_autobackup/ZfsAuto.py | 9 ++++---- 7 files changed, 47 insertions(+), 51 deletions(-) diff --git a/tests/test_encryption.py b/tests/test_encryption.py index 7b705bf..1a5ecbc 100644 --- a/tests/test_encryption.py +++ b/tests/test_encryption.py @@ -60,16 +60,16 @@ class TestZfsEncryption(unittest2.TestCase): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) + "test test_target1 --verbose --no-progress --allow-empty".split(" ")).run()) self.assertFalse(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split( + "test test_target1/encryptedtarget --verbose --no-progress --no-snapshot".split( " ")).run()) with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) + "test test_target1 --verbose --no-progress --allow-empty".split(" ")).run()) self.assertFalse(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split( + "test test_target1/encryptedtarget --verbose --no-progress --no-snapshot".split( " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") @@ -103,19 +103,19 @@ test_target1/test_source2/fs2/sub encryption with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split( + "test test_target1 --verbose --no-progress --decrypt --allow-empty".split( " ")).run()) # NOTE: this also tests if sending a source to a second target works correctly (this failed during bookmark implementation) self.assertFalse(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received --debug".split( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --debug".split( " ")).run()) with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split( + "test test_target1 --verbose --no-progress --decrypt --allow-empty".split( " ")).run()) self.assertFalse(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received".split( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot".split( " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") @@ -147,18 +147,18 @@ test_target1/test_source2/fs2/sub encryptionroot - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split( + "test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --clear-mountpoint".split( " ")).run()) self.assertFalse(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --clear-mountpoint".split( " ")).run()) with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split( + "test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --clear-mountpoint".split( " ")).run()) self.assertFalse(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --clear-mountpoint".split( " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") @@ -190,18 +190,18 @@ test_target1/test_source2/fs2/sub encryptionroot - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split( + "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --clear-mountpoint".split( " ")).run()) self.assertFalse(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --clear-mountpoint".split( " ")).run()) with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received".split( + "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty".split( " ")).run()) self.assertFalse(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --exclude-received".split( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot".split( " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") @@ -265,7 +265,7 @@ test_target1/test_source2/fs2/sub encryptionroot - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --exclude-received --clear-mountpoint".split( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --clear-mountpoint".split( " ")).run()) r = shelltest("zfs set compress=off test_source1 test_target1") @@ -279,7 +279,7 @@ test_target1/test_source2/fs2/sub encryptionroot - # should fail and leave resume token with mocktime("20101111000001"): self.assertTrue(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --clear-mountpoint".split( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --clear-mountpoint".split( " ")).run()) # # free up space @@ -296,7 +296,7 @@ test_target1/test_source2/fs2/sub encryptionroot - # resume should fail with mocktime("20101111000001"): self.assertEqual(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint".split( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --no-snapshot --clear-mountpoint".split( " ")).run(), 3) # NOTE: On some versions this leaves 2 weird sub-datasets that should'nt be there (its probably a zfs bug?) @@ -330,5 +330,5 @@ test_target1/test_source2/fs2/sub encryptionroot - # resume should complete with mocktime("20101111000001"): self.assertEqual(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint --debug".split( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --no-snapshot --clear-mountpoint --debug".split( " ")).run(), 0) diff --git a/tests/test_sendrecvpipes.py b/tests/test_sendrecvpipes.py index 5c7b4f5..bfd93e1 100644 --- a/tests/test_sendrecvpipes.py +++ b/tests/test_sendrecvpipes.py @@ -16,7 +16,7 @@ class TestSendRecvPipes(unittest2.TestCase): with self.subTest("local local pipe"): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", "--clear-mountpoint", "--send-pipe=dd bs=1M", "--recv-pipe=dd bs=2M"]).run()) @@ -25,7 +25,7 @@ class TestSendRecvPipes(unittest2.TestCase): with self.subTest("remote local pipe"): with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", "--ssh-source=localhost", "--send-pipe=dd bs=1M", "--recv-pipe=dd bs=2M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -33,7 +33,7 @@ class TestSendRecvPipes(unittest2.TestCase): with self.subTest("local remote pipe"): with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", "--ssh-target=localhost", "--send-pipe=dd bs=1M", "--recv-pipe=dd bs=2M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -41,7 +41,7 @@ class TestSendRecvPipes(unittest2.TestCase): with self.subTest("remote remote pipe"): with mocktime("20101111000003"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", "--ssh-source=localhost", "--ssh-target=localhost", "--send-pipe=dd bs=1M", "--recv-pipe=dd bs=2M"]).run()) @@ -75,7 +75,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 with self.subTest("compress " + compress): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--exclude-received", "--no-holds", "--no-progress", "--verbose", + ["test", "test_target1", "--no-holds", "--no-progress", "--verbose", "--compress=" + compress]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -86,7 +86,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 with self.subTest("local local pipe"): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", "--clear-mountpoint", "--buffer=1M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -94,7 +94,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 with self.subTest("remote local pipe"): with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--verbose", "--exclude-received", "--no-holds", + ["test", "test_target1", "--allow-empty", "--verbose", "--no-holds", "--no-progress", "--ssh-source=localhost", "--buffer=1M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -102,7 +102,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 with self.subTest("local remote pipe"): with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", "--ssh-target=localhost", "--buffer=1M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -110,7 +110,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 with self.subTest("remote remote pipe"): with mocktime("20101111000003"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", "--ssh-source=localhost", "--ssh-target=localhost", "--buffer=1M"]).run()) r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") @@ -142,7 +142,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 start = time.time() with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--exclude-received", "--no-holds", "--no-progress", "--rate=50k"]).run()) + ["test", "test_target1", "--no-holds", "--no-progress", "--rate=50k"]).run()) # not a great way of verifying but it works. self.assertGreater(time.time() - start, 5) diff --git a/tests/test_verify.py b/tests/test_verify.py index 5ecd402..c4164b7 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -69,7 +69,7 @@ class TestZfsVerify(unittest2.TestCase): self.assertFalse(ZfsAutoverify("test test_target1 --verbose --test".split(" ")).run()) with self.subTest("rsync, remote source and target. (not supported, all 6 fail)"): - self.assertEqual(6, ZfsAutoverify("test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --exclude-received --fs-compare=rsync".split(" ")).run()) + self.assertEqual(6, ZfsAutoverify("test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --fs-compare=rsync".split(" ")).run()) def runchecked(testname, command): with self.subTest(testname): @@ -83,20 +83,20 @@ class TestZfsVerify(unittest2.TestCase): self.assertRegex(buf.getvalue(), "bad_filesystem: FAILED:") self.assertRegex(buf.getvalue(), "bad_zvol: FAILED:") - runchecked("rsync, remote source", "test test_target1 --ssh-source=localhost --verbose --exclude-received --fs-compare=rsync") - runchecked("rsync, remote target", "test test_target1 --ssh-target=localhost --verbose --exclude-received --fs-compare=rsync") - runchecked("rsync, local", "test test_target1 --verbose --exclude-received --fs-compare=rsync") + runchecked("rsync, remote source", "test test_target1 --ssh-source=localhost --verbose --fs-compare=rsync") + runchecked("rsync, remote target", "test test_target1 --ssh-target=localhost --verbose --fs-compare=rsync") + runchecked("rsync, local", "test test_target1 --verbose --fs-compare=rsync") runchecked("tar, remote source and remote target", - "test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --exclude-received --fs-compare=find") + "test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --fs-compare=find") runchecked("tar, remote source", - "test test_target1 --ssh-source=localhost --verbose --exclude-received --fs-compare=find") + "test test_target1 --ssh-source=localhost --verbose --fs-compare=find") runchecked("tar, remote target", - "test test_target1 --ssh-target=localhost --verbose --exclude-received --fs-compare=find") - runchecked("tar, local", "test test_target1 --verbose --exclude-received --fs-compare=find") + "test test_target1 --ssh-target=localhost --verbose --fs-compare=find") + runchecked("tar, local", "test test_target1 --verbose --fs-compare=find") with self.subTest("no common snapshot"): #destroy common snapshot, now 3 should fail shelltest("zfs destroy test_source1/fs1/ok_zvol@test-20101111000000") - self.assertEqual(3, ZfsAutoverify("test test_target1 --verbose --exclude-received".split(" ")).run()) + self.assertEqual(3, ZfsAutoverify("test test_target1 --verbose ".split(" ")).run()) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 8eee85f..485b378 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -746,12 +746,12 @@ test_target1/test_source2/fs2/sub@test-20101111000003 with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --allow-empty --ssh-source localhost --exclude-received".split( + "test test_target1 --no-progress --verbose --allow-empty --ssh-source localhost".split( " ")).run()) with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --allow-empty --ssh-target localhost --exclude-received".split( + "test test_target1 --no-progress --verbose --allow-empty --ssh-target localhost".split( " ")).run()) with mocktime("20101111000002"): diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py index eab2b7b..6a34cd1 100644 --- a/tests/test_zfsautobackup40.py +++ b/tests/test_zfsautobackup40.py @@ -678,8 +678,3 @@ test_target1/test_source1/fs1/sub@other2 test_target1/test_source1/fs1/sub@test2-20010203000000 """) - -# TODO -# - check if holds and no-holds function ok when bookmarks are enabled -# - does it remove hold when going from no-bookmark to bookmark mode? -# - delete bookmarks we have no other snapshots diff --git a/zfs_autobackup/CliBase.py b/zfs_autobackup/CliBase.py index aced9d6..68c7e45 100644 --- a/zfs_autobackup/CliBase.py +++ b/zfs_autobackup/CliBase.py @@ -10,7 +10,7 @@ class CliBase(object): Overridden in subclasses that add stuff for the specific programs.""" # also used by setup.py - VERSION = "4.0-beta2" + VERSION = "4.0-beta3" HEADER = "{} v{} - (c)2026 E.H.Eefting (edwin@datux.nl)".format(os.path.basename(sys.argv[0]), VERSION) def __init__(self, argv, print_arguments=True): diff --git a/zfs_autobackup/ZfsAuto.py b/zfs_autobackup/ZfsAuto.py index 958b901..7ab2d2d 100644 --- a/zfs_autobackup/ZfsAuto.py +++ b/zfs_autobackup/ZfsAuto.py @@ -36,6 +36,10 @@ class ZfsAuto(CliBase): self.warning("--ignore-replicated has been renamed, using --exclude-unchanged") args.exclude_unchanged = True + if args.exclude_received: + self.log.error("--exclude-received is no longer needed. zfs-autobackup now filters all the autobackup:... properties for newly received datasets to solve this problem. To get rid of existing properties on the target do this one time: 'zfs inherit -r autobackup:backup1 pool/backup1'") + sys.exit(255) + # Note: Before version v3.1-beta5, we always used exclude_received. This was a problem if you wanted to # replicate an existing backup to another host and use the same backupname/snapshots. However, exclude_received # may still need to be used to explicitly exclude a backup with the 'received' source property to avoid accidental @@ -136,11 +140,8 @@ class ZfsAuto(CliBase): group.add_argument('--ignore-replicated', action='store_true', help=argparse.SUPPRESS) group.add_argument('--exclude-unchanged', metavar='BYTES', default=0, type=int, help='Exclude datasets that have less than BYTES data changed since any last snapshot. (Use with proxmox HA replication)') - group.add_argument('--exclude-received', action='store_true', - help='Exclude datasets that have the origin of their autobackup: property as "received".', ) + group.add_argument('--exclude-received', action='store_true', help=argparse.SUPPRESS) - # group.add_argument('--include-received', action='store_true', - # help=argparse.SUPPRESS) def regex_argument_type(input_line): """Parses regex arguments into re.Pattern objects""" From 7c044e1cb913d18565190dfc1fbb38416b7e300a Mon Sep 17 00:00:00 2001 From: DatuX Date: Sun, 7 Jun 2026 12:17:42 +0200 Subject: [PATCH 107/135] remove dead exclude_received code --- tests/test_zfsnode.py | 11 ++++------- zfs_autobackup/ZfsAuto.py | 2 +- zfs_autobackup/ZfsAutobackup.py | 1 - zfs_autobackup/ZfsAutoverify.py | 1 - zfs_autobackup/ZfsContainer.py | 8 +------- zfs_autobackup/ZfsNode.py | 3 +-- 6 files changed, 7 insertions(+), 19 deletions(-) diff --git a/tests/test_zfsnode.py b/tests/test_zfsnode.py index 67fc1a8..ad7a4c4 100644 --- a/tests/test_zfsnode.py +++ b/tests/test_zfsnode.py @@ -19,7 +19,7 @@ class TestZfsNode(unittest2.TestCase): with self.subTest("first snapshot"): (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", - exclude_paths=[], exclude_received=False, + exclude_paths=[], exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-20101111000001", 100000) r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) @@ -40,7 +40,7 @@ test_target1 with self.subTest("second snapshot, no changes, no snapshot"): (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", - exclude_paths=[], exclude_received=False, + exclude_paths=[], exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-20101111000002", 1) r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) @@ -61,7 +61,7 @@ test_target1 with self.subTest("second snapshot, no changes, empty snapshot"): (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", - exclude_paths=[], exclude_received=False, + exclude_paths=[], exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-20101111000002", 0) r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) @@ -95,7 +95,6 @@ test_target1 with redirect_stdout(buf): (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", exclude_paths=[], - exclude_received=False, exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-1", 0, @@ -115,7 +114,6 @@ test_target1 with self.assertRaises(ExecuteError): (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", exclude_paths=[], - exclude_received=False, exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-1", 0, @@ -136,7 +134,6 @@ test_target1 # same snapshot name as before so it fails (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", exclude_paths=[], - exclude_received=False, exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-1", 0, @@ -189,7 +186,7 @@ test_target1 logger=logger, description=description, debug_output=False, ssh_to=None, ssh_config=None, exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", - exclude_paths=[], exclude_received=False, + exclude_paths=[], exclude_unchanged=1) s = pformat(selected_datasets) print(s) diff --git a/zfs_autobackup/ZfsAuto.py b/zfs_autobackup/ZfsAuto.py index 7ab2d2d..cd63646 100644 --- a/zfs_autobackup/ZfsAuto.py +++ b/zfs_autobackup/ZfsAuto.py @@ -45,7 +45,7 @@ class ZfsAuto(CliBase): # may still need to be used to explicitly exclude a backup with the 'received' source property to avoid accidental # recursive replication of a zvol that is currently being received in another session (as it will have changes). - # Follow up note: This isnt a problem anymore since v3.4, we now filter the autobackup-property by default to prevent these difficult issues. + # Follow up note: This isnt a problem anymore since v4.0, we now filter the autobackup-property by default to prevent these difficult issues. self.exclude_paths = [] if args.ssh_source == args.ssh_target: diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 0552a81..acd8320 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -514,7 +514,6 @@ class ZfsAutobackup(ZfsAuto): ################# select source datasets self.set_title("Selecting") (source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, - exclude_received=self.args.exclude_received, exclude_paths=self.exclude_paths, exclude_unchanged=self.args.exclude_unchanged) if not source_datasets and not excluded_datasets: diff --git a/zfs_autobackup/ZfsAutoverify.py b/zfs_autobackup/ZfsAutoverify.py index 906ae0a..3dabdbf 100644 --- a/zfs_autobackup/ZfsAutoverify.py +++ b/zfs_autobackup/ZfsAutoverify.py @@ -240,7 +240,6 @@ class ZfsAutoverify(ZfsAuto): ################# select source datasets self.set_title("Selecting") (source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, - exclude_received=self.args.exclude_received, exclude_paths=self.exclude_paths, exclude_unchanged=self.args.exclude_unchanged) if not source_datasets and not excluded_datasets: diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index d31c65d..9f9ae00 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -538,7 +538,7 @@ class ZfsContainer(ZfsDataset): if target_snapshot.exists: target_snapshot.destroy() - def is_selected(self, value, source, inherited, exclude_received, exclude_paths, exclude_unchanged): + def is_selected(self, value, source, inherited, exclude_paths, exclude_unchanged): """determine if dataset should be selected for backup (called from ZfsNode) @@ -547,7 +547,6 @@ class ZfsContainer(ZfsDataset): :type value: str :type source: str :type inherited: bool - :type exclude_received: bool :type exclude_unchanged: int :param value: Value of the zfs property ("false"/"true"/"child"/parent/"-") @@ -597,11 +596,6 @@ class ZfsContainer(ZfsDataset): self.debug("Excluded (path in exclude list)") return False - if source == "received": - if exclude_received: - self.verbose("Excluded (dataset already received)") - return False - if not self.is_changed(exclude_unchanged): self.verbose("Excluded (by --exclude-unchanged)") return False diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index 5c65843..093e6e6 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -291,7 +291,7 @@ class ZfsNode(ExecuteNode): except Exception as e: pass - def selected_datasets(self, property_name, exclude_received, exclude_paths, exclude_unchanged): + def selected_datasets(self, property_name, exclude_paths, exclude_unchanged): """determine filesystems that should be backed up by looking at the special autobackup-property, systemwide returns: ( list of selected ZfsContainer, list of excluded ZfsContainer) @@ -328,7 +328,6 @@ class ZfsNode(ExecuteNode): # determine it selected = dataset.is_selected(value=value, source=source, inherited=inherited, - exclude_received=exclude_received, exclude_paths=exclude_paths, exclude_unchanged=exclude_unchanged) if selected == True: From 1076904765dca030d965a420f775a83d788ce888 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sun, 7 Jun 2026 12:26:01 +0200 Subject: [PATCH 108/135] test the new autobackup: property filtering --- tests/test_zfsautobackup40.py | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py index 6a34cd1..42d0f82 100644 --- a/tests/test_zfsautobackup40.py +++ b/tests/test_zfsautobackup40.py @@ -678,3 +678,41 @@ test_target1/test_source1/fs1/sub@other2 test_target1/test_source1/fs1/sub@test2-20010203000000 """) + def test_filter_autobackup_properties(self): + """All autobackup:* properties from the source must be filtered during + replication, and properties set directly on the target must not be + touched by subsequent backups.""" + + # an extra autobackup:* property on the source (besides the default autobackup:test=true on test_source1/fs1) + shelltest("zfs set autobackup:extra=sourceonly test_source1/fs1") + + # first backup, creates target hierarchy + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose".split(" ")).run()) + + # set autobackup:* properties directly on the target. + # autobackup:test deliberately collides with the source property to verify the source + # value does not overwrite the target on subsequent receives. + shelltest("zfs set autobackup:targetonly=targetvalue test_target1/test_source1/fs1") + shelltest("zfs set autobackup:test=false test_target1/test_source1/fs1/sub") + + properties = "autobackup:test,autobackup:extra,autobackup:targetonly" + before = shelltest( + "zfs get -H -o name,property,value,source -r -s local,received -t filesystem " + + properties + " test_target1") + + # second backup, must not change any autobackup:* property on the target + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + after = shelltest( + "zfs get -H -o name,property,value,source -r -s local,received -t filesystem " + + properties + " test_target1") + + self.assertMultiLineEqual(before, after) + + # autobackup:extra was only set on the source and must not have leaked to the target + self.assertNotIn("sourceonly", after) + From 5137f51662026b1465421c43e9e6554383f79615 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sun, 7 Jun 2026 14:35:41 +0200 Subject: [PATCH 109/135] more progress ouptut --- zfs_autobackup/LogConsole.py | 2 ++ zfs_autobackup/ZfsAutobackup.py | 3 +++ zfs_autobackup/ZfsContainer.py | 18 ++++++++++++++++++ zfs_autobackup/ZfsSnapshot.py | 3 +++ 4 files changed, 26 insertions(+) diff --git a/zfs_autobackup/LogConsole.py b/zfs_autobackup/LogConsole.py index fd1aa99..e036f77 100644 --- a/zfs_autobackup/LogConsole.py +++ b/zfs_autobackup/LogConsole.py @@ -2,6 +2,7 @@ from __future__ import print_function import sys +import time try: import colorama @@ -72,3 +73,4 @@ class LogConsole: else: print(self._CLEAR_LINE, end='', file=sys.stderr) self._progress_uncleared = False + sys.stderr.flush() diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index acd8320..280160e 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -513,6 +513,9 @@ class ZfsAutobackup(ZfsAuto): ################# select source datasets self.set_title("Selecting") + if self.args.progress: + self.progress(f"Selecting datasets with property {self.property_name}") + (source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, exclude_paths=self.exclude_paths, exclude_unchanged=self.args.exclude_unchanged) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 9f9ae00..51e1731 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -641,6 +641,9 @@ class ZfsContainer(ZfsDataset): # self.verbose("-> {}".format(target_dataset)) + if show_progress: + self.zfs_node.logger.progress("Planning sync..") + # defaults for these settings if there is no encryption stuff going on: send_properties = True raw = False @@ -656,23 +659,35 @@ class ZfsContainer(ZfsDataset): # keep data encrypted by sending it raw (including properties) raw = True + (source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots) = \ self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, guid_check=guid_check, raw=raw, bookmark_tag=bookmark_tag) + if show_progress: + self.zfs_node.logger.progress("Pre-cleaning..") + # NOTE: we do a pre-clean because we dont want filesystems to fillup when backups keep failing. # Also usefull with no_send to still cleanup stuff. self._pre_clean( source_common_snapshot=source_common_snapshot, target_dataset=target_dataset, target_transfers=target_transfers, target_obsoletes=target_obsoletes, source_obsoletes=source_obsoletes) + # check if we can resume if len(target_transfers)>0: + + if show_progress: + self.zfs_node.logger.progress("Verifying resume token...") + resume_token = self._validate_resume_token(target_dataset, target_transfers[0]) else: resume_token = None + if show_progress: + self.zfs_node.logger.progress("Preparing...") + # handle incompatible stuff on target target_dataset.handle_incompatible_target(incompatible_target_snapshots, destroy_incompatible, force, source_common_snapshot, resume_token is not None) @@ -707,6 +722,9 @@ class ZfsContainer(ZfsDataset): # do the rollback, one time at first transfer if do_rollback: + if show_progress: + self.zfs_node.logger.progress(f"Rolling back {target_dataset}...") + target_dataset.rollback() do_rollback = False diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index 9854954..cc7b47b 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -229,6 +229,9 @@ class ZfsSnapshot(ZfsPointInTime): # incremental self.verbose("-> {}".format(target_snapshot.prefix)) + if show_progress: + self.zfs_node.logger.progress("Transferring...") + # do it pipe = self.send_pipe(features=features, show_progress=show_progress, prev_snapshot=prev_snapshot, resume_token=resume_token, raw=raw, send_properties=send_properties, From 06f4849d05c6fe0b5c0c19e6810c7a684da1e835 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sun, 7 Jun 2026 17:34:39 +0200 Subject: [PATCH 110/135] better progress --- zfs_autobackup/CliBase.py | 4 ++-- zfs_autobackup/LogConsole.py | 28 ++++++++++++++++++++++++++-- zfs_autobackup/ZfsAutobackup.py | 8 ++++---- zfs_autobackup/ZfsContainer.py | 2 -- zfs_autobackup/ZfsNode.py | 2 +- 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/zfs_autobackup/CliBase.py b/zfs_autobackup/CliBase.py index 68c7e45..2bac095 100644 --- a/zfs_autobackup/CliBase.py +++ b/zfs_autobackup/CliBase.py @@ -99,8 +99,8 @@ class CliBase(object): def debug(self, txt): self.log.debug(txt) - def progress(self, txt): - self.log.progress(txt) + def progress(self, txt, current=None, total=None, failed=None): + self.log.progress(txt, current, total, failed) def clear_progress(self): self.log.clear_progress() diff --git a/zfs_autobackup/LogConsole.py b/zfs_autobackup/LogConsole.py index e036f77..51b66c9 100644 --- a/zfs_autobackup/LogConsole.py +++ b/zfs_autobackup/LogConsole.py @@ -25,6 +25,10 @@ class LogConsole: self.colorama = color and _COLORAMA_AVAILABLE + self._failed=None + self._current=None + self._total=None + def error(self, txt): self.clear_progress() if self.colorama: @@ -59,11 +63,31 @@ class LogConsole: print("# " + txt) sys.stdout.flush() - def progress(self, txt): + def progress(self, txt, current=None, total=None, failed=None): + + if current is not None: + self._current=current + + if total is not None: + self._total=total + + if failed is not None: + self._failed=failed + """print progress output to stderr (stays on same line)""" self.clear_progress() self._progress_uncleared = True - print(">>> {}\r".format(txt), end='', file=sys.stderr) + + if self._current is None: + line=f">>> {txt}" + else: + if self._failed: + line=f">>> [{self._current}/{self._total}] {txt} ({self._failed} failed)" + else: + line=f">>> [{self._current}/{self._total}] {txt}" + + print(line+"\r", end='', file=sys.stderr) + sys.stderr.flush() def clear_progress(self): diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 280160e..3dd9d22 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -174,7 +174,7 @@ class ZfsAutobackup(ZfsAuto): count = count + 1 if self.args.progress: - self.progress("Analysing missing {}/{}".format(count, len(missing_datasets))) + self.progress("Analysing missing..",count, len(missing_datasets), 0) try: dataset.debug("Missing on source, thinning") @@ -204,7 +204,7 @@ class ZfsAutobackup(ZfsAuto): count = count + 1 if self.args.progress: - self.progress("Analysing destroy missing {}/{}".format(count, len(missing_datasets))) + self.progress("Analysing destroy missing...", count, len(missing_datasets),0) try: # cant do anything without our own snapshots @@ -384,7 +384,7 @@ class ZfsAutobackup(ZfsAuto): # stats if self.args.progress: count = count + 1 - self.progress("Analysing dataset {}/{} ({} failed)".format(count, len(source_datasets), fail_count)) + self.progress("Analysing dataset...", count, len(source_datasets), fail_count) try: # determine corresponding target_dataset @@ -514,7 +514,7 @@ class ZfsAutobackup(ZfsAuto): ################# select source datasets self.set_title("Selecting") if self.args.progress: - self.progress(f"Selecting datasets with property {self.property_name}") + self.progress(f"Loading...") (source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, exclude_paths=self.exclude_paths, diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 51e1731..47b1962 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -641,8 +641,6 @@ class ZfsContainer(ZfsDataset): # self.verbose("-> {}".format(target_dataset)) - if show_progress: - self.zfs_node.logger.progress("Planning sync..") # defaults for these settings if there is no encryption stuff going on: send_properties = True diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index 093e6e6..7f0cc0e 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -202,7 +202,7 @@ class ZfsNode(ExecuteNode): minutes_left = int((bytes_left / (bytes_ / (time.time() - self._progress_start_time))) / 60) self.logger.progress( - "Transfer {}% {}MB/s (total {}MB, {} minutes left)".format(percentage, speed, int( + "Transfering {}% {}MB/s (total {}MB, {} minutes left)".format(percentage, speed, int( self._progress_total_bytes / (1024 * 1024)), minutes_left)) return From bf8b3d526c5cf38d49f13fa57a9f3732b76396ae Mon Sep 17 00:00:00 2001 From: DatuX Date: Sun, 7 Jun 2026 20:14:54 +0200 Subject: [PATCH 111/135] initial clone support, needs review --- tests/test_clone.py | 165 ++++++++++++++++++++++++++++++++ zfs_autobackup/ZfsAutobackup.py | 66 ++++++++++++- zfs_autobackup/ZfsContainer.py | 60 +++++++++++- zfs_autobackup/ZfsSnapshot.py | 9 +- 4 files changed, 296 insertions(+), 4 deletions(-) create mode 100644 tests/test_clone.py diff --git a/tests/test_clone.py b/tests/test_clone.py new file mode 100644 index 0000000..6f15ca4 --- /dev/null +++ b/tests/test_clone.py @@ -0,0 +1,165 @@ +from basetest import * + + +class TestZfsAutobackupClone(unittest2.TestCase): + """ZFS clone replication: source datasets that are themselves clones get + replicated to the target as clones of the corresponding target-side origin + snapshot, via 'zfs send -i @'.""" + + def setUp(self): + prepare_zpools() + self.longMessage = True + + def test_clone_happy_path(self): + """Single run, both origin and clone selected. + --other-snapshots ensures the origin snapshot itself reaches the target, + so the clone can pin to it.""" + + shelltest("zfs snapshot test_source1/fs1@base") + shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") + shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --other-snapshots".split(" ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@base +test_source1/fs1@test-20101111000000 +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000000 +test_source1/fs1_clone +test_source1/fs1_clone@test-20101111000000 +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000000 +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@base +test_target1/test_source1/fs1@test-20101111000000 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000000 +test_target1/test_source1/fs1_clone +test_target1/test_source1/fs1_clone@test-20101111000000 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000000 +""") + + origin = shelltest("zfs get -H -o value origin test_target1/test_source1/fs1_clone").strip() + self.assertEqual(origin, "test_target1/test_source1/fs1@base") + + def test_clone_origin_not_on_target_falls_back_to_full_send(self): + """If the clone's origin isn't replicated, warn and fall back to full send.""" + + shelltest("zfs inherit autobackup:test test_source1/fs1") + shelltest("zfs snapshot test_source1/fs1@base") + shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") + shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + + with OutputIO() as buf: + with redirect_stdout(buf), redirect_stderr(buf): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose".split(" ")).run()) + + output = buf.getvalue() + print(output) + self.assertIn("Cannot replicate as clone", output) + + origin = shelltest("zfs get -H -o value origin test_target1/test_source1/fs1_clone").strip() + self.assertEqual(origin, "-") + + def test_clone_after_initial_origin_backup(self): + """First run backs up the origin; second run replicates a clone of it.""" + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose".split(" ")).run()) + + shelltest("zfs clone test_source1/fs1@test-20101111000000 test_source1/fs1_clone") + shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + origin = shelltest("zfs get -H -o value origin test_target1/test_source1/fs1_clone").strip() + self.assertEqual(origin, "test_target1/test_source1/fs1@test-20101111000000") + + def test_no_clone_flag_disables_clone_replication(self): + """--no-clone reverts to the previous full-send behaviour for clones.""" + + shelltest("zfs snapshot test_source1/fs1@base") + shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") + shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --other-snapshots --no-clone".split(" ")).run()) + + origin = shelltest("zfs get -H -o value origin test_target1/test_source1/fs1_clone").strip() + self.assertEqual(origin, "-") + + def test_subsequent_incremental_after_clone_send(self): + """A second run after a clone has been sent is a plain incremental.""" + + shelltest("zfs snapshot test_source1/fs1@base") + shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") + shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --other-snapshots".split(" ")).run()) + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + r = shelltest("zfs list -H -o name -t snapshot test_target1/test_source1/fs1_clone") + self.assertMultiLineEqual(r, """ +test_target1/test_source1/fs1_clone@test-20101111000000 +test_target1/test_source1/fs1_clone@test-20101111000001 +""") + + origin = shelltest("zfs get -H -o value origin test_target1/test_source1/fs1_clone").strip() + self.assertEqual(origin, "test_target1/test_source1/fs1@base") + + def test_topological_sort_processes_origin_before_clone(self): + """When the clone alphabetically precedes its origin, the topo sort still + processes the origin first so the clone can pin to its target origin.""" + + shelltest("zfs snapshot test_source1/fs1@base") + shelltest("zfs clone test_source1/fs1@base test_source1/aaa_clone") + shelltest("zfs set autobackup:test=true test_source1/aaa_clone") + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --other-snapshots".split(" ")).run()) + + origin = shelltest("zfs get -H -o value origin test_target1/test_source1/aaa_clone").strip() + self.assertEqual(origin, "test_target1/test_source1/fs1@base") + + def test_test_mode_makes_no_changes_for_clones(self): + """--test must not modify anything, even when clone handling is exercised.""" + + shelltest("zfs snapshot test_source1/fs1@base") + shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") + shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --test --other-snapshots".split(" ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") + self.assertMultiLineEqual(r, """ +test_target1 +""") diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 3dd9d22..f8d04ff 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -92,6 +92,9 @@ class ZfsAutobackup(ZfsAuto): help='Don\'t hold snapshots. (Faster. Allows you to destroy common snapshot.)') group.add_argument('--no-bookmarks', action='store_true', help='Don\'t use bookmarks.') + group.add_argument('--no-clone', action='store_true', + help="Don't preserve clone relationships during replication. " + "Source datasets that are clones will be replicated as full standalone datasets.") group.add_argument('--clear-refreservation', action='store_true', help='Filter "refreservation" property. (recommended, saves space. same as ' @@ -345,6 +348,53 @@ class ZfsAutobackup(ZfsAuto): else: return self.args.target_path + def _topological_sort_for_clones(self, source_datasets): + """Reorder source_datasets so an origin is processed before any selected clone of it. + Datasets whose origin is not in the selection (or who are not clones) are independent + and keep their relative order. On a detected cycle (should not happen with valid ZFS) + the original list is returned with a warning. + """ + + by_name = {d.name: d for d in source_datasets} + + parent_dep = {} # dataset_name -> parent_dataset_name in selection, or None + for d in source_datasets: + try: + origin = d.properties.get('origin', '-') + except Exception: + origin = '-' + if origin == '-' or '@' not in origin: + parent_dep[d.name] = None + continue + parent_path = origin.split('@', 1)[0] + if parent_path in by_name and parent_path != d.name: + parent_dep[d.name] = parent_path + else: + parent_dep[d.name] = None + + indegree = {name: 0 for name in by_name} + children = {name: [] for name in by_name} + for name, parent in parent_dep.items(): + if parent is not None: + indegree[name] += 1 + children[parent].append(name) + + queue = [d.name for d in source_datasets if indegree[d.name] == 0] + result = [] + while queue: + name = queue.pop(0) + result.append(by_name[name]) + for child in children[name]: + indegree[child] -= 1 + if indegree[child] == 0: + queue.append(child) + + if len(result) != len(source_datasets): + self.warning("Clone topological sort: cycle detected, keeping original dataset order.") + return source_datasets + + return result + def check_target_names(self, source_node, source_datasets, target_node): """check all target names for collesions etc due to strip-options""" @@ -376,6 +426,9 @@ class ZfsAutobackup(ZfsAuto): send_pipes = self.get_send_pipes(source_node.verbose) recv_pipes = self.get_recv_pipes(target_node.verbose) + if not self.args.no_clone: + source_datasets = self._topological_sort_for_clones(source_datasets) + fail_count = 0 count = 0 target_datasets = [] @@ -418,6 +471,16 @@ class ZfsAutobackup(ZfsAuto): else: use_bookmarks = True + # if the source is a clone and the target-side origin is available, send + # the first snapshot as an incremental from the origin so the clone + # relationship is preserved on the target. + clone_origin_snapshot = None + if not self.args.no_clone: + clone_origin_snapshot = source_dataset.resolve_clone_origin( + target_node=target_node, + make_target_name=self.make_target_name, + guid_check=not self.args.no_guid_check) + # sync the snapshots of this dataset source_dataset.sync_snapshots(target_dataset, show_progress=self.args.progress, features=common_features, filter_properties=self.filter_properties_list(), @@ -432,7 +495,8 @@ class ZfsAutobackup(ZfsAuto): zfs_compressed=self.args.zfs_compressed, force=self.args.force, guid_check=not self.args.no_guid_check, use_bookmarks=use_bookmarks, bookmark_tag=bookmark_tag, - property_format=self.args.property_format) + property_format=self.args.property_format, + clone_origin_snapshot=clone_origin_snapshot) except Exception as e: fail_count = fail_count + 1 diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 47b1962..b0d65cb 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -608,10 +608,59 @@ class ZfsContainer(ZfsDataset): self.verbose("Destroying") return super().destroy(fail_exception=fail_exception) + def resolve_clone_origin(self, target_node, make_target_name, guid_check): + """If this dataset is a ZFS clone, return the source-side origin snapshot + suitable for use as zfs send -i when the corresponding target-side + origin exists. Returns None when the dataset is not a clone, when the target + origin is missing, when guids do not match, or when the strip-path mapping + does not apply. + + Args: + :type target_node: ZfsNode + :type make_target_name: callable taking a ZfsContainer, returning str + :type guid_check: bool + :rtype: ZfsSnapshot|None + """ + + origin = self.properties.get('origin', '-') + if origin == '-': + return None + + if '@' not in origin: + self.warning("Cannot replicate as clone: cannot parse origin '{}'.".format(origin)) + return None + + parent_path, snap_name = origin.split('@', 1) + source_origin_parent = self.zfs_node.get_dataset(parent_path) + + try: + target_origin_parent = make_target_name(source_origin_parent) + except Exception as e: + self.warning("Cannot replicate as clone: cannot map origin '{}' to target ({}). Falling back to full send.".format(origin, str(e))) + return None + + source_origin_snap = self.zfs_node.get_dataset(origin) + target_origin_snap = target_node.get_dataset(target_origin_parent + '@' + snap_name) + + if not target_origin_snap.exists: + self.warning("Cannot replicate as clone: origin '{}' not available on target. Falling back to full send.".format(origin)) + return None + + if guid_check: + try: + if source_origin_snap.properties.get('guid') != target_origin_snap.properties.get('guid'): + self.warning("Cannot replicate as clone: origin guid mismatch between source and target. Falling back to full send.") + return None + except ExecuteError: + # properties not readable (e.g. test mode after a simulated recv) — skip the check + self.debug("Cannot read origin guid; skipping guid check for clone replication.") + + return source_origin_snap + def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, - use_bookmarks, bookmark_tag, property_format): + use_bookmarks, bookmark_tag, property_format, clone_origin_snapshot=None): """sync this dataset's snapshots to target_dataset, while also thinning out old snapshots along the way. @@ -637,6 +686,7 @@ class ZfsContainer(ZfsDataset): :type use_bookmarks: bool :type bookmark_tag: str :type property_format: str + :type clone_origin_snapshot: ZfsSnapshot|None """ # self.verbose("-> {}".format(target_dataset)) @@ -711,7 +761,13 @@ class ZfsContainer(ZfsDataset): # now actually transfer the snapshots do_rollback = rollback - prev_source_snapshot_bookmark = source_common_snapshot + # When the target dataset is being created fresh and the source is a clone whose + # origin's target equivalent exists, send the first snapshot as an incremental from + # that origin so zfs recv reconstructs the clone relationship on the target. + if source_common_snapshot is None and clone_origin_snapshot is not None: + prev_source_snapshot_bookmark = clone_origin_snapshot + else: + prev_source_snapshot_bookmark = source_common_snapshot prev_target_snapshot = target_dataset.find_snapshot(source_common_snapshot) for target_snapshot in target_transfers: diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index cc7b47b..9a91547 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -51,6 +51,9 @@ class ZfsSnapshot(ZfsPointInTime): resume_token: resume sending from this token. (in that case we don't need to know snapshot names) + If prev_snapshot is on a different parent dataset (clone-of-origin case), + its full name is passed to zfs send -i; otherwise the short @suffix form is used. + Args: :param send_pipes: output cmd array that will be added to actual zfs send command. (e.g. mbuffer or compression program) :type send_pipes: list[str] @@ -99,7 +102,11 @@ class ZfsSnapshot(ZfsPointInTime): # incremental? if prev_snapshot: - cmd.extend(["-i", prev_snapshot.typed_suffix]) + if prev_snapshot.prefix != self.prefix: + # cross-dataset (clone replicated from its origin snapshot): full path required + cmd.extend(["-i", prev_snapshot.name]) + else: + cmd.extend(["-i", prev_snapshot.typed_suffix]) cmd.append(self.name) From 72c34da8ecce761c12d07d273252815678cdc57c Mon Sep 17 00:00:00 2001 From: DatuX Date: Sun, 7 Jun 2026 22:18:24 +0200 Subject: [PATCH 112/135] catch reverse clones --- tests/test_clone.py | 27 +++++++++++++++++++++++++++ zfs_autobackup/ZfsContainer.py | 9 +++++++++ 2 files changed, 36 insertions(+) diff --git a/tests/test_clone.py b/tests/test_clone.py index 6f15ca4..9e0e9c8 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -163,3 +163,30 @@ test_target1/test_source1/fs1_clone@test-20101111000001 self.assertMultiLineEqual(r, """ test_target1 """) + + def test_promoted_clone_origin_on_namespace_descendant_warns_and_falls_back(self): + """After 'zfs promote', the original parent becomes a clone whose origin lives + on one of its own namespace descendants. zfs recv can't land a clone-creating + stream on top of the placeholder we have to create for that descendant, so we + detect this upfront, warn, and fall back to a full send (which will itself + fail at the placeholder — user can destroy target and rerun, or promote the + source dataset to flip the lineage back).""" + + shelltest("zfs snapshot test_source1/fs1@s1") + shelltest("zfs clone test_source1/fs1@s1 test_source1/fs1/inner_clone") + shelltest("zfs promote test_source1/fs1/inner_clone") + + origin = shelltest("zfs get -H -o value origin test_source1/fs1").strip() + self.assertEqual(origin, "test_source1/fs1/inner_clone@s1") + + with OutputIO() as buf: + with redirect_stdout(buf), redirect_stderr(buf): + with mocktime("20101111000000"): + result = ZfsAutobackup( + "test test_target1 --no-progress --verbose --other-snapshots".split(" ")).run() + + output = buf.getvalue() + print(output) + + self.assertTrue(result) + self.assertIn("lives on a namespace descendant", output) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index b0d65cb..f44e96f 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -631,6 +631,15 @@ class ZfsContainer(ZfsDataset): return None parent_path, snap_name = origin.split('@', 1) + + # Reverse-clone topology (after 'zfs promote' of a child): origin lives on + # a namespace descendant of this dataset. zfs recv has no way to land a + # clone-creating stream on top of the placeholder that has to exist for + # the descendant, so replication can't preserve the relationship. + if parent_path == self.name or parent_path.startswith(self.name + "/"): + self.warning("Cannot replicate as clone: origin '{}' lives on a namespace descendant of this dataset. Falling back to full send.".format(origin)) + return None + source_origin_parent = self.zfs_node.get_dataset(parent_path) try: From 55c633a3e967fd497af0e330f4c0ee565c3ac23c Mon Sep 17 00:00:00 2001 From: DatuX Date: Sun, 7 Jun 2026 23:42:50 +0200 Subject: [PATCH 113/135] refactor clones --- zfs_autobackup/ZfsAutobackup.py | 46 +++++++++++++++++++++++++++++---- zfs_autobackup/ZfsContainer.py | 45 +++++--------------------------- 2 files changed, 48 insertions(+), 43 deletions(-) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index f8d04ff..1c56a52 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -2,7 +2,7 @@ import argparse from signal import signal, SIGPIPE from . import compressors -from .ExecuteNode import ExecuteNode +from .ExecuteNode import ExecuteNode, ExecuteError from .Thinner import Thinner from .ThinnerRule import ThinnerRule from .ZfsAuto import ZfsAuto @@ -348,11 +348,50 @@ class ZfsAutobackup(ZfsAuto): else: return self.args.target_path + def _resolve_clone_origin(self, source_dataset, target_node): + """Return the source-side origin snapshot to use as zfs send -i base for + a clone, or None if the clone relationship cannot be preserved on the target. + + :type source_dataset: ZfsContainer + :type target_node: ZfsNode + :rtype: ZfsSnapshot|None + """ + + + source_origin_snap = source_dataset.get_clone_origin_snapshot() + if source_origin_snap is None: + return None + + try: + target_origin_parent = self.make_target_name(source_origin_snap.parent) + except Exception as e: + source_dataset.warning("Cannot replicate as clone: cannot map origin '{}' to target ({}). Falling back to full send.".format(source_origin_snap.name, str(e))) + return None + + target_origin_snap = target_node.get_dataset(target_origin_parent + '@' + source_origin_snap.suffix) + + if not target_origin_snap.exists: + source_dataset.warning("Cannot replicate as clone: origin '{}' not available on target. Falling back to full send.".format(source_origin_snap.name)) + return None + + if not self.args.no_guid_check: + try: + if source_origin_snap.properties.get('guid') != target_origin_snap.properties.get('guid'): + source_dataset.warning("Cannot replicate as clone: origin guid mismatch between source and target. Falling back to full send.") + return None + except ExecuteError: + # properties not readable (e.g. test mode after a simulated recv) — skip the check + source_dataset.debug("Cannot read origin guid; skipping guid check for clone replication.") + + return source_origin_snap + def _topological_sort_for_clones(self, source_datasets): """Reorder source_datasets so an origin is processed before any selected clone of it. Datasets whose origin is not in the selection (or who are not clones) are independent and keep their relative order. On a detected cycle (should not happen with valid ZFS) the original list is returned with a warning. + + :rtype: list[ZfsContainer] """ by_name = {d.name: d for d in source_datasets} @@ -476,10 +515,7 @@ class ZfsAutobackup(ZfsAuto): # relationship is preserved on the target. clone_origin_snapshot = None if not self.args.no_clone: - clone_origin_snapshot = source_dataset.resolve_clone_origin( - target_node=target_node, - make_target_name=self.make_target_name, - guid_check=not self.args.no_guid_check) + clone_origin_snapshot = self._resolve_clone_origin(source_dataset, target_node) # sync the snapshots of this dataset source_dataset.sync_snapshots(target_dataset, show_progress=self.args.progress, diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index f44e96f..aa98307 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -2,7 +2,6 @@ from typing import cast from .ZfsBookmark import ZfsBookmark from .ZfsDataset import ZfsDataset -from .ExecuteNode import ExecuteError from .ZfsPointInTime import ZfsPointInTime from .ZfsSnapshot import ZfsSnapshot @@ -608,18 +607,12 @@ class ZfsContainer(ZfsDataset): self.verbose("Destroying") return super().destroy(fail_exception=fail_exception) - def resolve_clone_origin(self, target_node, make_target_name, guid_check): - """If this dataset is a ZFS clone, return the source-side origin snapshot - suitable for use as zfs send -i when the corresponding target-side - origin exists. Returns None when the dataset is not a clone, when the target - origin is missing, when guids do not match, or when the strip-path mapping - does not apply. + def get_clone_origin_snapshot(self): + """If this dataset is a ZFS clone whose origin can be replicated, return the + source-side origin snapshot. Returns None when the dataset is not a clone, + the origin cannot be parsed, or the clone topology is unreplicatable. - Args: - :type target_node: ZfsNode - :type make_target_name: callable taking a ZfsContainer, returning str - :type guid_check: bool - :rtype: ZfsSnapshot|None + :rtype: ZfsSnapshot|None """ origin = self.properties.get('origin', '-') @@ -630,7 +623,7 @@ class ZfsContainer(ZfsDataset): self.warning("Cannot replicate as clone: cannot parse origin '{}'.".format(origin)) return None - parent_path, snap_name = origin.split('@', 1) + parent_path = origin.split('@', 1)[0] # Reverse-clone topology (after 'zfs promote' of a child): origin lives on # a namespace descendant of this dataset. zfs recv has no way to land a @@ -640,31 +633,7 @@ class ZfsContainer(ZfsDataset): self.warning("Cannot replicate as clone: origin '{}' lives on a namespace descendant of this dataset. Falling back to full send.".format(origin)) return None - source_origin_parent = self.zfs_node.get_dataset(parent_path) - - try: - target_origin_parent = make_target_name(source_origin_parent) - except Exception as e: - self.warning("Cannot replicate as clone: cannot map origin '{}' to target ({}). Falling back to full send.".format(origin, str(e))) - return None - - source_origin_snap = self.zfs_node.get_dataset(origin) - target_origin_snap = target_node.get_dataset(target_origin_parent + '@' + snap_name) - - if not target_origin_snap.exists: - self.warning("Cannot replicate as clone: origin '{}' not available on target. Falling back to full send.".format(origin)) - return None - - if guid_check: - try: - if source_origin_snap.properties.get('guid') != target_origin_snap.properties.get('guid'): - self.warning("Cannot replicate as clone: origin guid mismatch between source and target. Falling back to full send.") - return None - except ExecuteError: - # properties not readable (e.g. test mode after a simulated recv) — skip the check - self.debug("Cannot read origin guid; skipping guid check for clone replication.") - - return source_origin_snap + return self.zfs_node.get_dataset(origin) def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, From 7f4506503af26339c28345061d952b30b96d8538 Mon Sep 17 00:00:00 2001 From: DatuX Date: Sun, 7 Jun 2026 23:54:40 +0200 Subject: [PATCH 114/135] automaticly include snapshots that are needed for cloning. --- tests/test_clone.py | 18 +++++++++--------- zfs_autobackup/ZfsAutobackup.py | 28 +++++++++++++++++++++++++++- zfs_autobackup/ZfsContainer.py | 13 +++++++++---- 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/tests/test_clone.py b/tests/test_clone.py index 9e0e9c8..574b302 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -11,9 +11,9 @@ class TestZfsAutobackupClone(unittest2.TestCase): self.longMessage = True def test_clone_happy_path(self): - """Single run, both origin and clone selected. - --other-snapshots ensures the origin snapshot itself reaches the target, - so the clone can pin to it.""" + """Single run, both origin and clone selected. The origin snapshot is + auto-included because a clone in the selection depends on it, so the user + doesn't need --other-snapshots.""" shelltest("zfs snapshot test_source1/fs1@base") shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") @@ -21,7 +21,7 @@ class TestZfsAutobackupClone(unittest2.TestCase): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --other-snapshots".split(" ")).run()) + "test test_target1 --no-progress --verbose".split(" ")).run()) r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertMultiLineEqual(r, """ @@ -104,7 +104,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --other-snapshots --no-clone".split(" ")).run()) + "test test_target1 --no-progress --verbose --no-clone".split(" ")).run()) origin = shelltest("zfs get -H -o value origin test_target1/test_source1/fs1_clone").strip() self.assertEqual(origin, "-") @@ -118,7 +118,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --other-snapshots".split(" ")).run()) + "test test_target1 --no-progress --verbose".split(" ")).run()) with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( @@ -143,7 +143,7 @@ test_target1/test_source1/fs1_clone@test-20101111000001 with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --other-snapshots".split(" ")).run()) + "test test_target1 --no-progress --verbose".split(" ")).run()) origin = shelltest("zfs get -H -o value origin test_target1/test_source1/aaa_clone").strip() self.assertEqual(origin, "test_target1/test_source1/fs1@base") @@ -157,7 +157,7 @@ test_target1/test_source1/fs1_clone@test-20101111000001 with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --test --other-snapshots".split(" ")).run()) + "test test_target1 --no-progress --verbose --test".split(" ")).run()) r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ @@ -183,7 +183,7 @@ test_target1 with redirect_stdout(buf), redirect_stderr(buf): with mocktime("20101111000000"): result = ZfsAutobackup( - "test test_target1 --no-progress --verbose --other-snapshots".split(" ")).run() + "test test_target1 --no-progress --verbose".split(" ")).run() output = buf.getvalue() print(output) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 1c56a52..c93e149 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -434,6 +434,28 @@ class ZfsAutobackup(ZfsAuto): return result + def _required_origin_snapshots(self, source_datasets): + """Build a map parent_dataset_name -> set of origin snapshot full names that + downstream clones (also in the selection) need pinned on the target. Used to + force-include those specific snapshots even without --other-snapshots. + + :rtype: dict[str, set[str]] + """ + + by_name = {d.name: d for d in source_datasets} + required = {} + for d in source_datasets: + try: + origin = d.properties.get('origin', '-') + except Exception: + origin = '-' + if origin == '-' or '@' not in origin: + continue + parent_path = origin.split('@', 1)[0] + if parent_path in by_name and parent_path != d.name: + required.setdefault(parent_path, set()).add(origin) + return required + def check_target_names(self, source_node, source_datasets, target_node): """check all target names for collesions etc due to strip-options""" @@ -467,6 +489,9 @@ class ZfsAutobackup(ZfsAuto): if not self.args.no_clone: source_datasets = self._topological_sort_for_clones(source_datasets) + required_origin_snapshots = self._required_origin_snapshots(source_datasets) + else: + required_origin_snapshots = {} fail_count = 0 count = 0 @@ -532,7 +557,8 @@ class ZfsAutobackup(ZfsAuto): guid_check=not self.args.no_guid_check, use_bookmarks=use_bookmarks, bookmark_tag=bookmark_tag, property_format=self.args.property_format, - clone_origin_snapshot=clone_origin_snapshot) + clone_origin_snapshot=clone_origin_snapshot, + required_snapshots=required_origin_snapshots.get(source_dataset.name)) except Exception as e: fail_count = fail_count + 1 diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index aa98307..ebc43eb 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -638,7 +638,8 @@ class ZfsContainer(ZfsDataset): def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, - use_bookmarks, bookmark_tag, property_format, clone_origin_snapshot=None): + use_bookmarks, bookmark_tag, property_format, clone_origin_snapshot=None, + required_snapshots=None): """sync this dataset's snapshots to target_dataset, while also thinning out old snapshots along the way. @@ -665,6 +666,7 @@ class ZfsContainer(ZfsDataset): :type bookmark_tag: str :type property_format: str :type clone_origin_snapshot: ZfsSnapshot|None + :type required_snapshots: set[str]|None """ # self.verbose("-> {}".format(target_dataset)) @@ -689,7 +691,8 @@ class ZfsContainer(ZfsDataset): (source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots) = \ self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, - guid_check=guid_check, raw=raw, bookmark_tag=bookmark_tag) + guid_check=guid_check, raw=raw, bookmark_tag=bookmark_tag, + required_snapshots=required_snapshots) if show_progress: self.zfs_node.logger.progress("Pre-cleaning..") @@ -872,7 +875,7 @@ class ZfsContainer(ZfsDataset): self.invalidate_cache() self.rollback() - def _plan_sync(self, target_dataset, also_other_snapshots, guid_check, raw, bookmark_tag): + def _plan_sync(self, target_dataset, also_other_snapshots, guid_check, raw, bookmark_tag, required_snapshots=None): """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. Args: @@ -882,6 +885,7 @@ class ZfsContainer(ZfsDataset): :type guid_check: bool :type raw: bool :type bookmark_tag: str + :type required_snapshots: set[str]|None Returns: tuple: A tuple containing: @@ -929,7 +933,8 @@ class ZfsContainer(ZfsDataset): while source_snapshot: # we want it? - if (also_other_snapshots or source_snapshot.is_ours) and not source_snapshot.is_snapshot_excluded: + is_required = required_snapshots is not None and source_snapshot.name in required_snapshots + if (also_other_snapshots or source_snapshot.is_ours or is_required) and not source_snapshot.is_snapshot_excluded: # create virtual target snapshot target_snapshot = cast(ZfsSnapshot, target_dataset.zfs_node.get_dataset( target_dataset.name + source_snapshot.typed_suffix, force_exists=False)) From c04c878367b5f07ab393cbdae765eda420cbffac Mon Sep 17 00:00:00 2001 From: DatuX Date: Mon, 8 Jun 2026 00:01:08 +0200 Subject: [PATCH 115/135] verbose log origin snapshot inclusion --- zfs_autobackup/ZfsContainer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index ebc43eb..53d093d 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -934,6 +934,8 @@ class ZfsContainer(ZfsDataset): while source_snapshot: # we want it? is_required = required_snapshots is not None and source_snapshot.name in required_snapshots + if is_required and not source_snapshot.is_ours and not also_other_snapshots: + source_snapshot.verbose("Including as clone origin for a selected clone") if (also_other_snapshots or source_snapshot.is_ours or is_required) and not source_snapshot.is_snapshot_excluded: # create virtual target snapshot target_snapshot = cast(ZfsSnapshot, target_dataset.zfs_node.get_dataset( From 5e81d4acecd41ca7ac41792c7bc0201893117997 Mon Sep 17 00:00:00 2001 From: DatuX Date: Mon, 8 Jun 2026 08:57:08 +0200 Subject: [PATCH 116/135] refactoring dataset_sync out of zfsautobackup --- zfs_autobackup/ZfsAutobackup.py | 317 +------------------------------- zfs_autobackup/ZfsNode.py | 1 + zfs_autobackup/sync_datasets.py | 311 +++++++++++++++++++++++++++++++ 3 files changed, 317 insertions(+), 312 deletions(-) create mode 100644 zfs_autobackup/sync_datasets.py diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index c93e149..b9fbbfc 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -2,12 +2,11 @@ import argparse from signal import signal, SIGPIPE from . import compressors -from .ExecuteNode import ExecuteNode, ExecuteError +from .ExecuteNode import ExecuteNode from .Thinner import Thinner -from .ThinnerRule import ThinnerRule from .ZfsAuto import ZfsAuto -from .ZfsContainer import ZfsContainer from .ZfsNode import ZfsNode +from .sync_datasets import sync_datasets from .util import sigpipe_handler, datetime_now @@ -160,104 +159,6 @@ class ZfsAutobackup(ZfsAuto): return parser - # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: - def thin_missing_targets(self, target_dataset, used_target_datasets): - """thin target datasets that are missing on the source. - :type used_target_datasets: list[ZfsContainer] - :type target_dataset: ZfsContainer - """ - - self.debug("Thinning obsolete datasets") - missing_datasets = [dataset for dataset in target_dataset.recursive_datasets if - dataset not in used_target_datasets] - - count = 0 - for dataset in missing_datasets: - self.debug("analyse missing {}".format(dataset)) - - count = count + 1 - if self.args.progress: - self.progress("Analysing missing..",count, len(missing_datasets), 0) - - try: - dataset.debug("Missing on source, thinning") - dataset.thin() - - except Exception as e: - dataset.error("Error during thinning of missing datasets ({})".format(str(e))) - - # if self.args.progress: - # self.clear_progress() - - # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: - def destroy_missing_targets(self, target_dataset, used_target_datasets): - """destroy target datasets that are missing on the source and that meet the requirements - :type used_target_datasets: list[ZfsContainer] - :type target_dataset: ZfsContainer - - """ - - self.debug("Destroying obsolete datasets") - - missing_datasets = [dataset for dataset in target_dataset.recursive_datasets if - dataset not in used_target_datasets] - - count = 0 - for dataset in missing_datasets: - - count = count + 1 - if self.args.progress: - self.progress("Analysing destroy missing...", count, len(missing_datasets),0) - - try: - # cant do anything without our own snapshots - if not dataset.our_snapshots: - if dataset.datasets: - # its not a leaf, just ignore - dataset.debug("Destroy missing: ignoring") - else: - dataset.verbose( - "Destroy missing: has no snapshots made by us (please destroy manually).") - else: - # past the deadline? - deadline_ttl = ThinnerRule("0s" + self.args.destroy_missing).ttl - now = datetime_now(self.args.utc).timestamp() - if dataset.our_snapshots[-1].timestamp + deadline_ttl > now: - dataset.verbose("Destroy missing: Waiting for deadline.") - else: - - dataset.debug("Destroy missing: Removing our snapshots.") - - # remove all our snaphots, except last, to safe space in case we fail later on - for snapshot in dataset.our_snapshots[:-1]: - snapshot.destroy(fail_exception=True) - - # does it have other snapshots? - has_others = False - for snapshot in dataset.snapshots: - if not snapshot.is_ours: - has_others = True - break - - if has_others: - dataset.verbose("Destroy missing: Still in use by other snapshots") - else: - if dataset.datasets: - dataset.verbose("Destroy missing: Still has children here.") - else: - dataset.verbose("Destroy missing.") - dataset.our_snapshots[-1].destroy(fail_exception=True) - dataset.destroy(fail_exception=True) - - except Exception as e: - # if self.args.progress: - # self.clear_progress() - - dataset.error("Error during --destroy-missing: {}".format(str(e))) - - # if self.args.progress: - # self.clear_progress() - def get_send_pipes(self, logger): """determine the zfs send pipe""" @@ -348,114 +249,6 @@ class ZfsAutobackup(ZfsAuto): else: return self.args.target_path - def _resolve_clone_origin(self, source_dataset, target_node): - """Return the source-side origin snapshot to use as zfs send -i base for - a clone, or None if the clone relationship cannot be preserved on the target. - - :type source_dataset: ZfsContainer - :type target_node: ZfsNode - :rtype: ZfsSnapshot|None - """ - - - source_origin_snap = source_dataset.get_clone_origin_snapshot() - if source_origin_snap is None: - return None - - try: - target_origin_parent = self.make_target_name(source_origin_snap.parent) - except Exception as e: - source_dataset.warning("Cannot replicate as clone: cannot map origin '{}' to target ({}). Falling back to full send.".format(source_origin_snap.name, str(e))) - return None - - target_origin_snap = target_node.get_dataset(target_origin_parent + '@' + source_origin_snap.suffix) - - if not target_origin_snap.exists: - source_dataset.warning("Cannot replicate as clone: origin '{}' not available on target. Falling back to full send.".format(source_origin_snap.name)) - return None - - if not self.args.no_guid_check: - try: - if source_origin_snap.properties.get('guid') != target_origin_snap.properties.get('guid'): - source_dataset.warning("Cannot replicate as clone: origin guid mismatch between source and target. Falling back to full send.") - return None - except ExecuteError: - # properties not readable (e.g. test mode after a simulated recv) — skip the check - source_dataset.debug("Cannot read origin guid; skipping guid check for clone replication.") - - return source_origin_snap - - def _topological_sort_for_clones(self, source_datasets): - """Reorder source_datasets so an origin is processed before any selected clone of it. - Datasets whose origin is not in the selection (or who are not clones) are independent - and keep their relative order. On a detected cycle (should not happen with valid ZFS) - the original list is returned with a warning. - - :rtype: list[ZfsContainer] - """ - - by_name = {d.name: d for d in source_datasets} - - parent_dep = {} # dataset_name -> parent_dataset_name in selection, or None - for d in source_datasets: - try: - origin = d.properties.get('origin', '-') - except Exception: - origin = '-' - if origin == '-' or '@' not in origin: - parent_dep[d.name] = None - continue - parent_path = origin.split('@', 1)[0] - if parent_path in by_name and parent_path != d.name: - parent_dep[d.name] = parent_path - else: - parent_dep[d.name] = None - - indegree = {name: 0 for name in by_name} - children = {name: [] for name in by_name} - for name, parent in parent_dep.items(): - if parent is not None: - indegree[name] += 1 - children[parent].append(name) - - queue = [d.name for d in source_datasets if indegree[d.name] == 0] - result = [] - while queue: - name = queue.pop(0) - result.append(by_name[name]) - for child in children[name]: - indegree[child] -= 1 - if indegree[child] == 0: - queue.append(child) - - if len(result) != len(source_datasets): - self.warning("Clone topological sort: cycle detected, keeping original dataset order.") - return source_datasets - - return result - - def _required_origin_snapshots(self, source_datasets): - """Build a map parent_dataset_name -> set of origin snapshot full names that - downstream clones (also in the selection) need pinned on the target. Used to - force-include those specific snapshots even without --other-snapshots. - - :rtype: dict[str, set[str]] - """ - - by_name = {d.name: d for d in source_datasets} - required = {} - for d in source_datasets: - try: - origin = d.properties.get('origin', '-') - except Exception: - origin = '-' - if origin == '-' or '@' not in origin: - continue - parent_path = origin.split('@', 1)[0] - if parent_path in by_name and parent_path != d.name: - required.setdefault(parent_path, set()).add(origin) - return required - def check_target_names(self, source_node, source_datasets, target_node): """check all target names for collesions etc due to strip-options""" @@ -475,108 +268,6 @@ class ZfsAutobackup(ZfsAuto): target_datasets[target_name] = source_dataset - # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: - def sync_datasets(self, source_node, source_datasets, target_node, bookmark_tag): - """Sync datasets, or thin-only on both sides - :type bookmark_tag: str - :type target_node: ZfsNode - :type source_datasets: list of ZfsContainer - :type source_node: ZfsNode - """ - - send_pipes = self.get_send_pipes(source_node.verbose) - recv_pipes = self.get_recv_pipes(target_node.verbose) - - if not self.args.no_clone: - source_datasets = self._topological_sort_for_clones(source_datasets) - required_origin_snapshots = self._required_origin_snapshots(source_datasets) - else: - required_origin_snapshots = {} - - fail_count = 0 - count = 0 - target_datasets = [] - for source_dataset in source_datasets: - - # stats - if self.args.progress: - count = count + 1 - self.progress("Analysing dataset...", count, len(source_datasets), fail_count) - - try: - # determine corresponding target_dataset - target_name = self.make_target_name(source_dataset) - target_dataset = target_node.get_dataset(target_name) - assert isinstance(target_dataset, ZfsContainer) - target_datasets.append(target_dataset) - - # ensure parents exists - # TODO: this isnt perfect yet, in some cases it can create parents when it shouldn't. - target_parent = target_dataset.parent - if not self.args.no_send \ - and target_parent is not None \ - and target_parent not in target_datasets \ - and not target_parent.exists: - target_dataset.debug("Creating unmountable parents") - target_parent.create_filesystem(parents=True) - - # determine common zpool features (cached, so no problem we call it often) - source_features = source_node.get_pool(source_dataset).features - target_features = target_node.get_pool(target_dataset).features - common_features = [f for f in source_features if f in target_features] - - if self.args.no_bookmarks: - use_bookmarks = False - else: - # NOTE: bookmark_written seems to be needed. (only 'bookmarks' was not enough on ubuntu 20) - if not 'bookmark_written' in common_features: - source_dataset.warning("Disabling bookmarks, not supported on both pools.") - use_bookmarks = False - else: - use_bookmarks = True - - # if the source is a clone and the target-side origin is available, send - # the first snapshot as an incremental from the origin so the clone - # relationship is preserved on the target. - clone_origin_snapshot = None - if not self.args.no_clone: - clone_origin_snapshot = self._resolve_clone_origin(source_dataset, target_node) - - # sync the snapshots of this dataset - source_dataset.sync_snapshots(target_dataset, show_progress=self.args.progress, - features=common_features, filter_properties=self.filter_properties_list(), - set_properties=self.set_properties_list(), - ignore_recv_exit_code=self.args.ignore_transfer_errors, - holds=not self.args.no_holds, rollback=self.args.rollback, - also_other_snapshots=self.args.other_snapshots, - no_send=self.args.no_send, - destroy_incompatible=self.args.destroy_incompatible, - send_pipes=send_pipes, recv_pipes=recv_pipes, - decrypt=self.args.decrypt, encrypt=self.args.encrypt, - zfs_compressed=self.args.zfs_compressed, force=self.args.force, - guid_check=not self.args.no_guid_check, use_bookmarks=use_bookmarks, - bookmark_tag=bookmark_tag, - property_format=self.args.property_format, - clone_origin_snapshot=clone_origin_snapshot, - required_snapshots=required_origin_snapshots.get(source_dataset.name)) - except Exception as e: - - fail_count = fail_count + 1 - source_dataset.error("FAILED: " + str(e)) - if self.args.debug: - self.verbose("Debug mode, aborting on first error") - raise - - target_path_dataset = target_node.get_dataset(self.args.target_path) - assert isinstance(target_path_dataset, ZfsContainer) - if not self.args.no_thinning: - self.thin_missing_targets(target_dataset=target_path_dataset, used_target_datasets=target_datasets) - - if self.args.destroy_missing is not None: - self.destroy_missing_targets(target_dataset=target_path_dataset, used_target_datasets=target_datasets) - - return fail_count - def thin_source(self, source_datasets): self.set_title("Thinning source") @@ -698,7 +389,9 @@ class ZfsAutobackup(ZfsAuto): # do the actual sync # NOTE: even with no_send, no_thinning and no_snapshot it does a usefull thing because it checks if the common snapshots and shows incompatible snapshots - fail_count = self.sync_datasets( + fail_count = sync_datasets( + self, + self.log, source_node=source_node, source_datasets=source_datasets, target_node=target_node, diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index 7f0cc0e..ff70596 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -15,6 +15,7 @@ from .ExecuteNode import ExecuteError from .util import datetime_now + class ZfsNode(ExecuteNode): """a node that contains zfs datasets. implements global (systemwide/pool wide) zfs commands""" diff --git a/zfs_autobackup/sync_datasets.py b/zfs_autobackup/sync_datasets.py new file mode 100644 index 0000000..0653bb1 --- /dev/null +++ b/zfs_autobackup/sync_datasets.py @@ -0,0 +1,311 @@ +from .ExecuteNode import ExecuteError +from .ThinnerRule import ThinnerRule +from .ZfsContainer import ZfsContainer +from .util import datetime_now + + +def thin_missing_targets(logger, target_dataset, used_target_datasets): + """thin target datasets that are missing on the source. + :type logger: LogConsole + :type used_target_datasets: list[ZfsContainer] + :type target_dataset: ZfsContainer + """ + + logger.debug("Thinning obsolete datasets") + missing_datasets = [dataset for dataset in target_dataset.recursive_datasets if + dataset not in used_target_datasets] + + count = 0 + for dataset in missing_datasets: + logger.debug("analyse missing {}".format(dataset)) + + count = count + 1 + logger.progress("Analysing missing..", count, len(missing_datasets), 0) + + try: + dataset.debug("Missing on source, thinning") + dataset.thin() + + except Exception as e: + dataset.error("Error during thinning of missing datasets ({})".format(str(e))) + + +# NOTE: this function also uses backup.args. args that need extra processing are passed as function parameters: +def destroy_missing_targets(backup, logger, target_dataset, used_target_datasets): + """destroy target datasets that are missing on the source and that meet the requirements + :type backup: ZfsAutobackup + :type logger: LogConsole + :type used_target_datasets: list[ZfsContainer] + :type target_dataset: ZfsContainer + + """ + + logger.debug("Destroying obsolete datasets") + + missing_datasets = [dataset for dataset in target_dataset.recursive_datasets if + dataset not in used_target_datasets] + + count = 0 + for dataset in missing_datasets: + + count = count + 1 + logger.progress("Analysing destroy missing...", count, len(missing_datasets), 0) + + try: + # cant do anything without our own snapshots + if not dataset.our_snapshots: + if dataset.datasets: + # its not a leaf, just ignore + dataset.debug("Destroy missing: ignoring") + else: + dataset.verbose( + "Destroy missing: has no snapshots made by us (please destroy manually).") + else: + # past the deadline? + deadline_ttl = ThinnerRule("0s" + backup.args.destroy_missing).ttl + now = datetime_now(backup.args.utc).timestamp() + if dataset.our_snapshots[-1].timestamp + deadline_ttl > now: + dataset.verbose("Destroy missing: Waiting for deadline.") + else: + + dataset.debug("Destroy missing: Removing our snapshots.") + + # remove all our snaphots, except last, to safe space in case we fail later on + for snapshot in dataset.our_snapshots[:-1]: + snapshot.destroy(fail_exception=True) + + # does it have other snapshots? + has_others = False + for snapshot in dataset.snapshots: + if not snapshot.is_ours: + has_others = True + break + + if has_others: + dataset.verbose("Destroy missing: Still in use by other snapshots") + else: + if dataset.datasets: + dataset.verbose("Destroy missing: Still has children here.") + else: + dataset.verbose("Destroy missing.") + dataset.our_snapshots[-1].destroy(fail_exception=True) + dataset.destroy(fail_exception=True) + + except Exception as e: + dataset.error("Error during --destroy-missing: {}".format(str(e))) + + +def _resolve_clone_origin(backup, source_dataset, target_node): + """Return the source-side origin snapshot to use as zfs send -i base for + a clone, or None if the clone relationship cannot be preserved on the target. + + :type backup: ZfsAutobackup + :type source_dataset: ZfsContainer + :type target_node: ZfsNode + :rtype: ZfsSnapshot|None + """ + + source_origin_snap = source_dataset.get_clone_origin_snapshot() + if source_origin_snap is None: + return None + + try: + target_origin_parent = backup.make_target_name(source_origin_snap.parent) + except Exception as e: + source_dataset.warning("Cannot replicate as clone: cannot map origin '{}' to target ({}). Falling back to full send.".format(source_origin_snap.name, str(e))) + return None + + target_origin_snap = target_node.get_dataset(target_origin_parent + '@' + source_origin_snap.suffix) + + if not target_origin_snap.exists: + source_dataset.warning("Cannot replicate as clone: origin '{}' not available on target. Falling back to full send.".format(source_origin_snap.name)) + return None + + if not backup.args.no_guid_check: + try: + if source_origin_snap.properties.get('guid') != target_origin_snap.properties.get('guid'): + source_dataset.warning("Cannot replicate as clone: origin guid mismatch between source and target. Falling back to full send.") + return None + except ExecuteError: + # properties not readable (e.g. test mode after a simulated recv) — skip the check + source_dataset.debug("Cannot read origin guid; skipping guid check for clone replication.") + + return source_origin_snap + + +def _topological_sort_for_clones(logger, source_datasets): + """Reorder source_datasets so an origin is processed before any selected clone of it. + Datasets whose origin is not in the selection (or who are not clones) are independent + and keep their relative order. On a detected cycle (should not happen with valid ZFS) + the original list is returned with a warning. + + :type logger: LogConsole + :rtype: list[ZfsContainer] + """ + + by_name = {d.name: d for d in source_datasets} + + parent_dep = {} # dataset_name -> parent_dataset_name in selection, or None + for d in source_datasets: + try: + origin = d.properties.get('origin', '-') + except Exception: + origin = '-' + if origin == '-' or '@' not in origin: + parent_dep[d.name] = None + continue + parent_path = origin.split('@', 1)[0] + if parent_path in by_name and parent_path != d.name: + parent_dep[d.name] = parent_path + else: + parent_dep[d.name] = None + + indegree = {name: 0 for name in by_name} + children = {name: [] for name in by_name} + for name, parent in parent_dep.items(): + if parent is not None: + indegree[name] += 1 + children[parent].append(name) + + queue = [d.name for d in source_datasets if indegree[d.name] == 0] + result = [] + while queue: + name = queue.pop(0) + result.append(by_name[name]) + for child in children[name]: + indegree[child] -= 1 + if indegree[child] == 0: + queue.append(child) + + if len(result) != len(source_datasets): + logger.warning("Clone topological sort: cycle detected, keeping original dataset order.") + return source_datasets + + return result + + +def _required_origin_snapshots(source_datasets): + """Build a map parent_dataset_name -> set of origin snapshot full names that + downstream clones (also in the selection) need pinned on the target. Used to + force-include those specific snapshots even without --other-snapshots. + + :rtype: dict[str, set[str]] + """ + + by_name = {d.name: d for d in source_datasets} + required = {} + for d in source_datasets: + try: + origin = d.properties.get('origin', '-') + except Exception: + origin = '-' + if origin == '-' or '@' not in origin: + continue + parent_path = origin.split('@', 1)[0] + if parent_path in by_name and parent_path != d.name: + required.setdefault(parent_path, set()).add(origin) + return required + + +# NOTE: this function also uses backup.args. args that need extra processing are passed as function parameters: +def sync_datasets(backup, logger, source_node, source_datasets, target_node, bookmark_tag): + """Sync datasets, or thin-only on both sides + :type backup: ZfsAutobackup + :type logger: LogConsole + :type bookmark_tag: str + :type target_node: ZfsNode + :type source_datasets: list of ZfsContainer + :type source_node: ZfsNode + """ + + send_pipes = backup.get_send_pipes(source_node.verbose) + recv_pipes = backup.get_recv_pipes(target_node.verbose) + + if not backup.args.no_clone: + source_datasets = _topological_sort_for_clones(logger, source_datasets) + required_origin_snapshots = _required_origin_snapshots(source_datasets) + else: + required_origin_snapshots = {} + + fail_count = 0 + count = 0 + target_datasets = [] + for source_dataset in source_datasets: + + # stats + count = count + 1 + logger.progress("Analysing dataset...", count, len(source_datasets), fail_count) + + try: + # determine corresponding target_dataset + target_name = backup.make_target_name(source_dataset) + target_dataset = target_node.get_dataset(target_name) + assert isinstance(target_dataset, ZfsContainer) + target_datasets.append(target_dataset) + + # ensure parents exists + # TODO: this isnt perfect yet, in some cases it can create parents when it shouldn't. + target_parent = target_dataset.parent + if not backup.args.no_send \ + and target_parent is not None \ + and target_parent not in target_datasets \ + and not target_parent.exists: + target_dataset.debug("Creating unmountable parents") + target_parent.create_filesystem(parents=True) + + # determine common zpool features (cached, so no problem we call it often) + source_features = source_node.get_pool(source_dataset).features + target_features = target_node.get_pool(target_dataset).features + common_features = [f for f in source_features if f in target_features] + + if backup.args.no_bookmarks: + use_bookmarks = False + else: + # NOTE: bookmark_written seems to be needed. (only 'bookmarks' was not enough on ubuntu 20) + if not 'bookmark_written' in common_features: + source_dataset.warning("Disabling bookmarks, not supported on both pools.") + use_bookmarks = False + else: + use_bookmarks = True + + # if the source is a clone and the target-side origin is available, send + # the first snapshot as an incremental from the origin so the clone + # relationship is preserved on the target. + clone_origin_snapshot = None + if not backup.args.no_clone: + clone_origin_snapshot = _resolve_clone_origin(backup, source_dataset, target_node) + + # sync the snapshots of this dataset + source_dataset.sync_snapshots(target_dataset, show_progress=backup.args.progress, + features=common_features, filter_properties=backup.filter_properties_list(), + set_properties=backup.set_properties_list(), + ignore_recv_exit_code=backup.args.ignore_transfer_errors, + holds=not backup.args.no_holds, rollback=backup.args.rollback, + also_other_snapshots=backup.args.other_snapshots, + no_send=backup.args.no_send, + destroy_incompatible=backup.args.destroy_incompatible, + send_pipes=send_pipes, recv_pipes=recv_pipes, + decrypt=backup.args.decrypt, encrypt=backup.args.encrypt, + zfs_compressed=backup.args.zfs_compressed, force=backup.args.force, + guid_check=not backup.args.no_guid_check, use_bookmarks=use_bookmarks, + bookmark_tag=bookmark_tag, + property_format=backup.args.property_format, + clone_origin_snapshot=clone_origin_snapshot, + required_snapshots=required_origin_snapshots.get(source_dataset.name)) + except Exception as e: + + fail_count = fail_count + 1 + source_dataset.error("FAILED: " + str(e)) + if backup.args.debug: + logger.verbose("Debug mode, aborting on first error") + raise + + target_path_dataset = target_node.get_dataset(backup.args.target_path) + assert isinstance(target_path_dataset, ZfsContainer) + if not backup.args.no_thinning: + thin_missing_targets(logger, target_dataset=target_path_dataset, used_target_datasets=target_datasets) + + if backup.args.destroy_missing is not None: + destroy_missing_targets(backup, logger, target_dataset=target_path_dataset, used_target_datasets=target_datasets) + + return fail_count From fa41f598f7435fc22f217d8a2000e7df7245aaab Mon Sep 17 00:00:00 2001 From: DatuX Date: Mon, 8 Jun 2026 09:09:23 +0200 Subject: [PATCH 117/135] refactoring dataset_sync out of zfsautobackup --- tests/test_thinner.py | 1 + zfs_autobackup/ZfsAutobackup.py | 27 ++++++- zfs_autobackup/sync_datasets.py | 123 ++++++++++++++++++++------------ 3 files changed, 102 insertions(+), 49 deletions(-) diff --git a/tests/test_thinner.py b/tests/test_thinner.py index 097ec32..93919d7 100644 --- a/tests/test_thinner.py +++ b/tests/test_thinner.py @@ -2,6 +2,7 @@ from basetest import * import pprint from zfs_autobackup.Thinner import Thinner +from zfs_autobackup.ThinnerRule import ThinnerRule # randint is different in python 2 vs 3 randint_compat = lambda lo, hi: lo + int(random.random() * (hi + 1 - lo)) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index b9fbbfc..e54b203 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -390,12 +390,35 @@ class ZfsAutobackup(ZfsAuto): # do the actual sync # NOTE: even with no_send, no_thinning and no_snapshot it does a usefull thing because it checks if the common snapshots and shows incompatible snapshots fail_count = sync_datasets( - self, self.log, source_node=source_node, source_datasets=source_datasets, target_node=target_node, - bookmark_tag=bookmark_tag) + bookmark_tag=bookmark_tag, + send_pipes=self.get_send_pipes(source_node.verbose), + recv_pipes=self.get_recv_pipes(target_node.verbose), + make_target_name=self.make_target_name, + no_clone=self.args.no_clone, + no_send=self.args.no_send, + no_bookmarks=self.args.no_bookmarks, + no_thinning=self.args.no_thinning, + filter_properties=self.filter_properties_list(), + set_properties=self.set_properties_list(), + ignore_transfer_errors=self.args.ignore_transfer_errors, + holds=not self.args.no_holds, + rollback=self.args.rollback, + other_snapshots=self.args.other_snapshots, + destroy_incompatible=self.args.destroy_incompatible, + decrypt=self.args.decrypt, + encrypt=self.args.encrypt, + zfs_compressed=self.args.zfs_compressed, + force=self.args.force, + guid_check=not self.args.no_guid_check, + property_format=self.args.property_format, + debug=self.args.debug, + target_path=self.args.target_path, + destroy_missing=self.args.destroy_missing, + utc=self.args.utc) # no target specified, run in snapshot-only mode else: diff --git a/zfs_autobackup/sync_datasets.py b/zfs_autobackup/sync_datasets.py index 0653bb1..0fc3dee 100644 --- a/zfs_autobackup/sync_datasets.py +++ b/zfs_autobackup/sync_datasets.py @@ -30,14 +30,13 @@ def thin_missing_targets(logger, target_dataset, used_target_datasets): dataset.error("Error during thinning of missing datasets ({})".format(str(e))) -# NOTE: this function also uses backup.args. args that need extra processing are passed as function parameters: -def destroy_missing_targets(backup, logger, target_dataset, used_target_datasets): +def destroy_missing_targets(logger, target_dataset, used_target_datasets, destroy_missing, utc): """destroy target datasets that are missing on the source and that meet the requirements - :type backup: ZfsAutobackup :type logger: LogConsole :type used_target_datasets: list[ZfsContainer] :type target_dataset: ZfsContainer - + :type destroy_missing: str + :type utc: bool """ logger.debug("Destroying obsolete datasets") @@ -62,8 +61,8 @@ def destroy_missing_targets(backup, logger, target_dataset, used_target_datasets "Destroy missing: has no snapshots made by us (please destroy manually).") else: # past the deadline? - deadline_ttl = ThinnerRule("0s" + backup.args.destroy_missing).ttl - now = datetime_now(backup.args.utc).timestamp() + deadline_ttl = ThinnerRule("0s" + destroy_missing).ttl + now = datetime_now(utc).timestamp() if dataset.our_snapshots[-1].timestamp + deadline_ttl > now: dataset.verbose("Destroy missing: Waiting for deadline.") else: @@ -95,11 +94,12 @@ def destroy_missing_targets(backup, logger, target_dataset, used_target_datasets dataset.error("Error during --destroy-missing: {}".format(str(e))) -def _resolve_clone_origin(backup, source_dataset, target_node): - """Return the source-side origin snapshot to use as zfs send -i base for - a clone, or None if the clone relationship cannot be preserved on the target. +def _resolve_clone_origin(make_target_name, guid_check, source_dataset, target_node): + """Return the source-side origin snapshot to use as zfs send -i base for a clone, + or None if the clone relationship cannot be preserved on the target. - :type backup: ZfsAutobackup + :type make_target_name: callable + :type guid_check: bool :type source_dataset: ZfsContainer :type target_node: ZfsNode :rtype: ZfsSnapshot|None @@ -110,7 +110,7 @@ def _resolve_clone_origin(backup, source_dataset, target_node): return None try: - target_origin_parent = backup.make_target_name(source_origin_snap.parent) + target_origin_parent = make_target_name(source_origin_snap.parent) except Exception as e: source_dataset.warning("Cannot replicate as clone: cannot map origin '{}' to target ({}). Falling back to full send.".format(source_origin_snap.name, str(e))) return None @@ -121,7 +121,7 @@ def _resolve_clone_origin(backup, source_dataset, target_node): source_dataset.warning("Cannot replicate as clone: origin '{}' not available on target. Falling back to full send.".format(source_origin_snap.name)) return None - if not backup.args.no_guid_check: + if guid_check: try: if source_origin_snap.properties.get('guid') != target_origin_snap.properties.get('guid'): source_dataset.warning("Cannot replicate as clone: origin guid mismatch between source and target. Falling back to full send.") @@ -140,6 +140,7 @@ def _topological_sort_for_clones(logger, source_datasets): the original list is returned with a warning. :type logger: LogConsole + :type source_datasets: list[ZfsContainer] :rtype: list[ZfsContainer] """ @@ -189,6 +190,7 @@ def _required_origin_snapshots(source_datasets): downstream clones (also in the selection) need pinned on the target. Used to force-include those specific snapshots even without --other-snapshots. + :type source_datasets: list[ZfsContainer] :rtype: dict[str, set[str]] """ @@ -207,21 +209,48 @@ def _required_origin_snapshots(source_datasets): return required -# NOTE: this function also uses backup.args. args that need extra processing are passed as function parameters: -def sync_datasets(backup, logger, source_node, source_datasets, target_node, bookmark_tag): - """Sync datasets, or thin-only on both sides - :type backup: ZfsAutobackup +def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_tag, + send_pipes, recv_pipes, make_target_name, + no_clone, no_send, no_bookmarks, no_thinning, + filter_properties, set_properties, + ignore_transfer_errors, holds, rollback, other_snapshots, + destroy_incompatible, decrypt, encrypt, zfs_compressed, force, + guid_check, property_format, debug, + target_path, destroy_missing, utc): + """Sync datasets, or thin-only on both sides. :type logger: LogConsole - :type bookmark_tag: str - :type target_node: ZfsNode - :type source_datasets: list of ZfsContainer :type source_node: ZfsNode + :type source_datasets: list[ZfsContainer] + :type target_node: ZfsNode + :type bookmark_tag: str + :type send_pipes: list + :type recv_pipes: list + :type make_target_name: callable + :type no_clone: bool + :type no_send: bool + :type no_bookmarks: bool + :type no_thinning: bool + :type filter_properties: list[str] + :type set_properties: list[str] + :type ignore_transfer_errors: bool + :type holds: bool + :type rollback: bool + :type other_snapshots: bool + :type destroy_incompatible: bool + :type decrypt: bool + :type encrypt: bool + :type zfs_compressed: bool + :type force: bool + :type guid_check: bool + :type property_format: str + :type debug: bool + :type target_path: str + :type destroy_missing: str|None + :type utc: bool + :rtype: int """ - send_pipes = backup.get_send_pipes(source_node.verbose) - recv_pipes = backup.get_recv_pipes(target_node.verbose) - - if not backup.args.no_clone: + if not no_clone: source_datasets = _topological_sort_for_clones(logger, source_datasets) required_origin_snapshots = _required_origin_snapshots(source_datasets) else: @@ -232,13 +261,12 @@ def sync_datasets(backup, logger, source_node, source_datasets, target_node, boo target_datasets = [] for source_dataset in source_datasets: - # stats count = count + 1 logger.progress("Analysing dataset...", count, len(source_datasets), fail_count) try: # determine corresponding target_dataset - target_name = backup.make_target_name(source_dataset) + target_name = make_target_name(source_dataset) target_dataset = target_node.get_dataset(target_name) assert isinstance(target_dataset, ZfsContainer) target_datasets.append(target_dataset) @@ -246,7 +274,7 @@ def sync_datasets(backup, logger, source_node, source_datasets, target_node, boo # ensure parents exists # TODO: this isnt perfect yet, in some cases it can create parents when it shouldn't. target_parent = target_dataset.parent - if not backup.args.no_send \ + if not no_send \ and target_parent is not None \ and target_parent not in target_datasets \ and not target_parent.exists: @@ -258,11 +286,11 @@ def sync_datasets(backup, logger, source_node, source_datasets, target_node, boo target_features = target_node.get_pool(target_dataset).features common_features = [f for f in source_features if f in target_features] - if backup.args.no_bookmarks: + if no_bookmarks: use_bookmarks = False else: # NOTE: bookmark_written seems to be needed. (only 'bookmarks' was not enough on ubuntu 20) - if not 'bookmark_written' in common_features: + if 'bookmark_written' not in common_features: source_dataset.warning("Disabling bookmarks, not supported on both pools.") use_bookmarks = False else: @@ -272,40 +300,41 @@ def sync_datasets(backup, logger, source_node, source_datasets, target_node, boo # the first snapshot as an incremental from the origin so the clone # relationship is preserved on the target. clone_origin_snapshot = None - if not backup.args.no_clone: - clone_origin_snapshot = _resolve_clone_origin(backup, source_dataset, target_node) + if not no_clone: + clone_origin_snapshot = _resolve_clone_origin(make_target_name, guid_check, source_dataset, target_node) # sync the snapshots of this dataset - source_dataset.sync_snapshots(target_dataset, show_progress=backup.args.progress, - features=common_features, filter_properties=backup.filter_properties_list(), - set_properties=backup.set_properties_list(), - ignore_recv_exit_code=backup.args.ignore_transfer_errors, - holds=not backup.args.no_holds, rollback=backup.args.rollback, - also_other_snapshots=backup.args.other_snapshots, - no_send=backup.args.no_send, - destroy_incompatible=backup.args.destroy_incompatible, + source_dataset.sync_snapshots(target_dataset, show_progress=True, + features=common_features, filter_properties=filter_properties, + set_properties=set_properties, + ignore_recv_exit_code=ignore_transfer_errors, + holds=holds, rollback=rollback, + also_other_snapshots=other_snapshots, + no_send=no_send, + destroy_incompatible=destroy_incompatible, send_pipes=send_pipes, recv_pipes=recv_pipes, - decrypt=backup.args.decrypt, encrypt=backup.args.encrypt, - zfs_compressed=backup.args.zfs_compressed, force=backup.args.force, - guid_check=not backup.args.no_guid_check, use_bookmarks=use_bookmarks, + decrypt=decrypt, encrypt=encrypt, + zfs_compressed=zfs_compressed, force=force, + guid_check=guid_check, use_bookmarks=use_bookmarks, bookmark_tag=bookmark_tag, - property_format=backup.args.property_format, + property_format=property_format, clone_origin_snapshot=clone_origin_snapshot, required_snapshots=required_origin_snapshots.get(source_dataset.name)) except Exception as e: fail_count = fail_count + 1 source_dataset.error("FAILED: " + str(e)) - if backup.args.debug: + if debug: logger.verbose("Debug mode, aborting on first error") raise - target_path_dataset = target_node.get_dataset(backup.args.target_path) + target_path_dataset = target_node.get_dataset(target_path) assert isinstance(target_path_dataset, ZfsContainer) - if not backup.args.no_thinning: + if not no_thinning: thin_missing_targets(logger, target_dataset=target_path_dataset, used_target_datasets=target_datasets) - if backup.args.destroy_missing is not None: - destroy_missing_targets(backup, logger, target_dataset=target_path_dataset, used_target_datasets=target_datasets) + if destroy_missing is not None: + destroy_missing_targets(logger, target_dataset=target_path_dataset, used_target_datasets=target_datasets, + destroy_missing=destroy_missing, utc=utc) return fail_count From d03bed9b4a40b400087f89ea9622065009bf8cde Mon Sep 17 00:00:00 2001 From: DatuX Date: Mon, 8 Jun 2026 09:21:17 +0200 Subject: [PATCH 118/135] refactoring dataset_sync out of zfsautobackup --- zfs_autobackup/ZfsAutobackup.py | 4 ++-- zfs_autobackup/ZfsNode.py | 27 +++++++++++++++++++++++++++ zfs_autobackup/sync_datasets.py | 15 ++++++--------- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index e54b203..05a0dc8 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -374,7 +374,7 @@ class ZfsAutobackup(ZfsAuto): target_node.verbose("Receive datasets under: {}".format(self.args.target_path)) # check if exists, to prevent vague errors - target_dataset = target_node.get_dataset(self.args.target_path) + target_dataset = target_node.get_container(self.args.target_path) if not target_dataset.exists: raise (Exception( "Target path '{}' does not exist. Please create this dataset first.".format(target_dataset))) @@ -416,7 +416,7 @@ class ZfsAutobackup(ZfsAuto): guid_check=not self.args.no_guid_check, property_format=self.args.property_format, debug=self.args.debug, - target_path=self.args.target_path, + target_dataset_base=target_dataset, destroy_missing=self.args.destroy_missing, utc=self.args.utc) diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index ff70596..1c4731e 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -150,6 +150,33 @@ class ZfsNode(ExecuteNode): return self._datasets[name] + def get_container(self, name, force_exists=None): + """get a ZfsContainer from name, asserting the name contains no @ or #. + :type name: str + :rtype: ZfsContainer + """ + dataset = self.get_dataset(name, force_exists=force_exists) + assert isinstance(dataset, ZfsContainer) + return dataset + + def get_snapshot(self, name, force_exists=None): + """get a ZfsSnapshot from name, asserting the name contains @. + :type name: str + :rtype: ZfsSnapshot + """ + dataset = self.get_dataset(name, force_exists=force_exists) + assert isinstance(dataset, ZfsSnapshot) + return dataset + + def get_bookmark(self, name, force_exists=None): + """get a ZfsBookmark from name, asserting the name contains #. + :type name: str + :rtype: ZfsBookmark + """ + dataset = self.get_dataset(name, force_exists=force_exists) + assert isinstance(dataset, ZfsBookmark) + return dataset + def get_datasets(self, names, force_exists=None): """get a list of ZfsDataset() object from names. stores objects internally to enable caching :rtype: list[ZfsContainer | ZfsBookmark | ZfsSnapshot] diff --git a/zfs_autobackup/sync_datasets.py b/zfs_autobackup/sync_datasets.py index 0fc3dee..f7d2260 100644 --- a/zfs_autobackup/sync_datasets.py +++ b/zfs_autobackup/sync_datasets.py @@ -115,7 +115,7 @@ def _resolve_clone_origin(make_target_name, guid_check, source_dataset, target_n source_dataset.warning("Cannot replicate as clone: cannot map origin '{}' to target ({}). Falling back to full send.".format(source_origin_snap.name, str(e))) return None - target_origin_snap = target_node.get_dataset(target_origin_parent + '@' + source_origin_snap.suffix) + target_origin_snap = target_node.get_snapshot(target_origin_parent + '@' + source_origin_snap.suffix) if not target_origin_snap.exists: source_dataset.warning("Cannot replicate as clone: origin '{}' not available on target. Falling back to full send.".format(source_origin_snap.name)) @@ -216,7 +216,7 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta ignore_transfer_errors, holds, rollback, other_snapshots, destroy_incompatible, decrypt, encrypt, zfs_compressed, force, guid_check, property_format, debug, - target_path, destroy_missing, utc): + target_dataset_base, destroy_missing, utc): """Sync datasets, or thin-only on both sides. :type logger: LogConsole :type source_node: ZfsNode @@ -244,7 +244,7 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta :type guid_check: bool :type property_format: str :type debug: bool - :type target_path: str + :type target_dataset: ZfsContainer :type destroy_missing: str|None :type utc: bool :rtype: int @@ -267,8 +267,7 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta try: # determine corresponding target_dataset target_name = make_target_name(source_dataset) - target_dataset = target_node.get_dataset(target_name) - assert isinstance(target_dataset, ZfsContainer) + target_dataset = target_node.get_container(target_name) target_datasets.append(target_dataset) # ensure parents exists @@ -328,13 +327,11 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta logger.verbose("Debug mode, aborting on first error") raise - target_path_dataset = target_node.get_dataset(target_path) - assert isinstance(target_path_dataset, ZfsContainer) if not no_thinning: - thin_missing_targets(logger, target_dataset=target_path_dataset, used_target_datasets=target_datasets) + thin_missing_targets(logger, target_dataset=target_dataset_base, used_target_datasets=target_datasets) if destroy_missing is not None: - destroy_missing_targets(logger, target_dataset=target_path_dataset, used_target_datasets=target_datasets, + destroy_missing_targets(logger, target_dataset=target_dataset_base, used_target_datasets=target_datasets, destroy_missing=destroy_missing, utc=utc) return fail_count From 6bd0570e472780422d2bebdd56d43d4188e5617a Mon Sep 17 00:00:00 2001 From: DatuX Date: Mon, 8 Jun 2026 09:57:56 +0200 Subject: [PATCH 119/135] use specif getter instead of get_dataset --- zfs_autobackup/ZfsAutoverify.py | 2 +- zfs_autobackup/ZfsCheck.py | 6 +++--- zfs_autobackup/ZfsContainer.py | 8 ++++---- zfs_autobackup/ZfsNode.py | 7 +++---- zfs_autobackup/ZfsPointInTime.py | 2 +- zfs_autobackup/ZfsSnapshot.py | 4 ++-- 6 files changed, 14 insertions(+), 15 deletions(-) diff --git a/zfs_autobackup/ZfsAutoverify.py b/zfs_autobackup/ZfsAutoverify.py index 3dabdbf..1b502d7 100644 --- a/zfs_autobackup/ZfsAutoverify.py +++ b/zfs_autobackup/ZfsAutoverify.py @@ -182,7 +182,7 @@ class ZfsAutoverify(ZfsAuto): try: # determine corresponding target_dataset target_name = self.make_target_name(source_dataset) - target_dataset = target_node.get_dataset(target_name) + target_dataset = target_node.get_container(target_name) # find common snapshots to verify source_snapshot = source_dataset.find_common_snapshot(target_dataset, True) diff --git a/zfs_autobackup/ZfsCheck.py b/zfs_autobackup/ZfsCheck.py index f1254ac..dab17a1 100644 --- a/zfs_autobackup/ZfsCheck.py +++ b/zfs_autobackup/ZfsCheck.py @@ -130,7 +130,7 @@ class ZfsCheck(CliBase): """ clone_name = get_tmp_clone_name(snapshot) - clone = snapshot.zfs_node.get_dataset(clone_name) + clone = snapshot.zfs_node.get_container(clone_name) clone.destroy() def generate_tree_hashes(self, prepared_target): @@ -243,7 +243,7 @@ class ZfsCheck(CliBase): if "@" in self.args.target: # zfs snapshot - snapshot = self.node.get_dataset(self.args.target) + snapshot = self.node.get_snapshot(self.args.target) if not snapshot.exists: raise Exception("ZFS snapshot {} does not exist!".format(snapshot)) dataset_type = snapshot.parent.properties['type'] @@ -259,7 +259,7 @@ class ZfsCheck(CliBase): def cleanup_target(self): if "@" in self.args.target: # zfs snapshot - snapshot = self.node.get_dataset(self.args.target) + snapshot = self.node.get_snapshot(self.args.target) if not snapshot.exists: return diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 53d093d..d45c053 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -29,7 +29,7 @@ class ZfsContainer(ZfsDataset): """ stripped = self.rstrip_path(1) if stripped: - return self.zfs_node.get_dataset(stripped) + return self.zfs_node.get_container(stripped) else: return None @@ -633,7 +633,7 @@ class ZfsContainer(ZfsDataset): self.warning("Cannot replicate as clone: origin '{}' lives on a namespace descendant of this dataset. Falling back to full send.".format(origin)) return None - return self.zfs_node.get_dataset(origin) + return self.zfs_node.get_snapshot(origin) def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, @@ -938,8 +938,8 @@ class ZfsContainer(ZfsDataset): source_snapshot.verbose("Including as clone origin for a selected clone") if (also_other_snapshots or source_snapshot.is_ours or is_required) and not source_snapshot.is_snapshot_excluded: # create virtual target snapshot - target_snapshot = cast(ZfsSnapshot, target_dataset.zfs_node.get_dataset( - target_dataset.name + source_snapshot.typed_suffix, force_exists=False)) + target_snapshot = target_dataset.zfs_node.get_snapshot( + target_dataset.name + source_snapshot.typed_suffix, force_exists=False) possible_target_snapshots.append(target_snapshot) source_snapshot = self.find_next_snapshot(source_snapshot) diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index 1c4731e..f516b7f 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -276,8 +276,7 @@ class ZfsNode(ExecuteNode): continue # force_exist, since we're making it - snapshot = self.get_dataset(dataset.name + "@" + snapshot_name, force_exists=True) - assert(isinstance(snapshot,ZfsSnapshot)) + snapshot = self.get_snapshot(dataset.name + "@" + snapshot_name, force_exists=True) if self.readonly: snapshot.simulate_properties() @@ -342,7 +341,7 @@ class ZfsNode(ExecuteNode): for line in lines: (name, value, raw_source) = line - dataset = self.get_dataset(name, force_exists=True) + dataset = self.get_container(name, force_exists=True) # "resolve" inherited sources sources[name] = raw_source @@ -385,7 +384,7 @@ class ZfsNode(ExecuteNode): for line in lines: matches = re.findall("toname = (.*@.*)", line) if matches: - snapshot = self.get_dataset(matches[0]) + snapshot = self.get_snapshot(matches[0]) snapshot.debug("resume token belongs to this snapshot") return snapshot diff --git a/zfs_autobackup/ZfsPointInTime.py b/zfs_autobackup/ZfsPointInTime.py index 52333cf..7325016 100644 --- a/zfs_autobackup/ZfsPointInTime.py +++ b/zfs_autobackup/ZfsPointInTime.py @@ -36,7 +36,7 @@ class ZfsPointInTime(ZfsDataset): :rtype: ZfsContainer """ - return self.zfs_node.get_dataset(self.prefix) + return self.zfs_node.get_container(self.prefix) @property diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index 9a91547..1c7e700 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -274,7 +274,7 @@ class ZfsSnapshot(ZfsPointInTime): self.zfs_node.run(cmd=cmd) - bookmark = self.zfs_node.get_dataset(bookmark_name, force_exists=True) + bookmark = self.zfs_node.get_bookmark(bookmark_name, force_exists=True) self.parent.cache_snapshot_bookmark(bookmark) return bookmark @@ -325,5 +325,5 @@ class ZfsSnapshot(ZfsPointInTime): self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - return self.zfs_node.get_dataset(name, force_exists=True) + return self.zfs_node.get_container(name, force_exists=True) From 750de74be5ccecd43c808300924932f11e34052e Mon Sep 17 00:00:00 2001 From: DatuX Date: Mon, 8 Jun 2026 10:13:31 +0200 Subject: [PATCH 120/135] refactor make_path --- zfs_autobackup/ZfsAuto.py | 3 --- zfs_autobackup/ZfsAutobackup.py | 13 +++---------- zfs_autobackup/ZfsAutoverify.py | 3 +-- zfs_autobackup/ZfsContainer.py | 12 ++++++++++++ zfs_autobackup/sync_datasets.py | 16 +++++++++------- 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/zfs_autobackup/ZfsAuto.py b/zfs_autobackup/ZfsAuto.py index cd63646..b9b7035 100644 --- a/zfs_autobackup/ZfsAuto.py +++ b/zfs_autobackup/ZfsAuto.py @@ -160,6 +160,3 @@ class ZfsAuto(CliBase): "No source filesystems selected, please do a 'zfs set {}=true' on the source datasets " "you want to select.".format(self.property_name)) - def make_target_name(self, source_dataset): - """make target_name from a source_dataset""" - return self.args.target_path + "/" + source_dataset.lstrip_path(self.args.strip_path) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 05a0dc8..bc483a9 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -241,14 +241,6 @@ class ZfsAutobackup(ZfsAuto): return ret - def make_target_name(self, source_dataset): - """make target_name from a source_dataset""" - stripped = source_dataset.lstrip_path(self.args.strip_path) - if stripped != "": - return self.args.target_path + "/" + stripped - else: - return self.args.target_path - def check_target_names(self, source_node, source_datasets, target_node): """check all target names for collesions etc due to strip-options""" @@ -256,7 +248,7 @@ class ZfsAutobackup(ZfsAuto): target_datasets = {} for source_dataset in source_datasets: - target_name = self.make_target_name(source_dataset) + target_name = source_dataset.map_to_target_path(self.args.target_path, self.args.strip_path) source_dataset.debug("-> {}".format(target_name)) if target_name in target_datasets: @@ -397,7 +389,8 @@ class ZfsAutobackup(ZfsAuto): bookmark_tag=bookmark_tag, send_pipes=self.get_send_pipes(source_node.verbose), recv_pipes=self.get_recv_pipes(target_node.verbose), - make_target_name=self.make_target_name, + target_path=self.args.target_path, + strip_path=self.args.strip_path, no_clone=self.args.no_clone, no_send=self.args.no_send, no_bookmarks=self.args.no_bookmarks, diff --git a/zfs_autobackup/ZfsAutoverify.py b/zfs_autobackup/ZfsAutoverify.py index 1b502d7..0f75732 100644 --- a/zfs_autobackup/ZfsAutoverify.py +++ b/zfs_autobackup/ZfsAutoverify.py @@ -181,8 +181,7 @@ class ZfsAutoverify(ZfsAuto): try: # determine corresponding target_dataset - target_name = self.make_target_name(source_dataset) - target_dataset = target_node.get_container(target_name) + target_dataset = target_node.get_container(source_dataset.map_to_target_path(self.args.target_path, self.args.strip_path)) # find common snapshots to verify source_snapshot = source_dataset.find_common_snapshot(target_dataset, True) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index d45c053..fb5cf2b 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -33,6 +33,18 @@ class ZfsContainer(ZfsDataset): else: return None + def map_to_target_path(self, target_path, strip_path): + """Map this dataset's name to a target path by stripping strip_path components and prepending target_path. + :type target_path: str + :type strip_path: int + :rtype: str + """ + stripped = self.lstrip_path(strip_path) + if stripped: + return target_path + "/" + stripped + else: + return target_path + def invalidate_cache(self): super().invalidate_cache() self._written_since_ours = None diff --git a/zfs_autobackup/sync_datasets.py b/zfs_autobackup/sync_datasets.py index f7d2260..93e8efb 100644 --- a/zfs_autobackup/sync_datasets.py +++ b/zfs_autobackup/sync_datasets.py @@ -94,11 +94,12 @@ def destroy_missing_targets(logger, target_dataset, used_target_datasets, destro dataset.error("Error during --destroy-missing: {}".format(str(e))) -def _resolve_clone_origin(make_target_name, guid_check, source_dataset, target_node): +def _resolve_clone_origin(target_path, strip_path, guid_check, source_dataset, target_node): """Return the source-side origin snapshot to use as zfs send -i base for a clone, or None if the clone relationship cannot be preserved on the target. - :type make_target_name: callable + :type target_path: str + :type strip_path: int :type guid_check: bool :type source_dataset: ZfsContainer :type target_node: ZfsNode @@ -110,7 +111,7 @@ def _resolve_clone_origin(make_target_name, guid_check, source_dataset, target_n return None try: - target_origin_parent = make_target_name(source_origin_snap.parent) + target_origin_parent = source_origin_snap.parent.map_to_target_path(target_path, strip_path) except Exception as e: source_dataset.warning("Cannot replicate as clone: cannot map origin '{}' to target ({}). Falling back to full send.".format(source_origin_snap.name, str(e))) return None @@ -210,7 +211,7 @@ def _required_origin_snapshots(source_datasets): def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_tag, - send_pipes, recv_pipes, make_target_name, + send_pipes, recv_pipes, target_path, strip_path, no_clone, no_send, no_bookmarks, no_thinning, filter_properties, set_properties, ignore_transfer_errors, holds, rollback, other_snapshots, @@ -225,7 +226,8 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta :type bookmark_tag: str :type send_pipes: list :type recv_pipes: list - :type make_target_name: callable + :type target_path: str + :type strip_path: int :type no_clone: bool :type no_send: bool :type no_bookmarks: bool @@ -266,7 +268,7 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta try: # determine corresponding target_dataset - target_name = make_target_name(source_dataset) + target_name = source_dataset.map_to_target_path(target_path, strip_path) target_dataset = target_node.get_container(target_name) target_datasets.append(target_dataset) @@ -300,7 +302,7 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta # relationship is preserved on the target. clone_origin_snapshot = None if not no_clone: - clone_origin_snapshot = _resolve_clone_origin(make_target_name, guid_check, source_dataset, target_node) + clone_origin_snapshot = _resolve_clone_origin(target_path, strip_path, guid_check, source_dataset, target_node) # sync the snapshots of this dataset source_dataset.sync_snapshots(target_dataset, show_progress=True, From d66e4bd60a50df8cd17f48fe16640de13aae2512 Mon Sep 17 00:00:00 2001 From: DatuX Date: Mon, 8 Jun 2026 10:34:54 +0200 Subject: [PATCH 121/135] refactored sync_snapshots to seperate module --- zfs_autobackup/ZfsContainer.py | 378 ------------------------------- zfs_autobackup/sync_datasets.py | 3 +- zfs_autobackup/sync_snapshots.py | 378 +++++++++++++++++++++++++++++++ 3 files changed, 380 insertions(+), 379 deletions(-) create mode 100644 zfs_autobackup/sync_snapshots.py diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index fb5cf2b..3d06bf4 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -522,33 +522,6 @@ class ZfsContainer(ZfsDataset): return None, None - def _pre_clean(self, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): - """cleanup old stuff on the source before starting snapshot syncing - - Args: - :type source_common_snapshot: ZfsSnapshot|ZfsBookmark|None - :type target_dataset: ZfsContainer - :type source_obsoletes: list[ZfsSnapshot] - :type target_obsoletes: list[ZfsSnapshot] - :type target_transfers: list[ZfsSnapshot] - """ - - # on source: delete all obsoletes that are not in target_transfers (except common snapshot, if its not a bookmark) - for source_snapshot in self.snapshots: - if (source_snapshot in source_obsoletes - and source_common_snapshot != source_snapshot - and source_snapshot.find_snapshot_by_suffix(target_transfers) is None): - source_snapshot.destroy() - - # on target: destroy everything thats obsolete, except common_snapshot - if target_dataset.exists: - for target_snapshot in target_dataset.snapshots: - if (target_snapshot in target_obsoletes) \ - and (not source_common_snapshot or ( - target_snapshot.tagless_suffix != source_common_snapshot.tagless_suffix)): - if target_snapshot.exists: - target_snapshot.destroy() - def is_selected(self, value, source, inherited, exclude_paths, exclude_unchanged): """determine if dataset should be selected for backup (called from ZfsNode) @@ -646,354 +619,3 @@ class ZfsContainer(ZfsDataset): return None return self.zfs_node.get_snapshot(origin) - - def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, - ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, - no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, - use_bookmarks, bookmark_tag, property_format, clone_origin_snapshot=None, - required_snapshots=None): - """sync this dataset's snapshots to target_dataset, while also thinning - out old snapshots along the way. - - Args: - :type send_pipes: list[str] - :type recv_pipes: list[str] - :type target_dataset: ZfsContainer - :type features: list[str] - :type show_progress: bool - :type filter_properties: list[str] - :type set_properties: list[str] - :type ignore_recv_exit_code: bool - :type holds: bool - :type rollback: bool - :type decrypt: bool - :type encrypt: bool - :type also_other_snapshots: bool - :type no_send: bool - :type destroy_incompatible: bool - :type zfs_compressed: bool - :type force: bool - :type guid_check: bool - :type use_bookmarks: bool - :type bookmark_tag: str - :type property_format: str - :type clone_origin_snapshot: ZfsSnapshot|None - :type required_snapshots: set[str]|None - """ - - # self.verbose("-> {}".format(target_dataset)) - - - # defaults for these settings if there is no encryption stuff going on: - send_properties = True - raw = False - write_embedded = True - - # source dataset encrypted? - if self.properties.get('encryption', 'off') != 'off': - # user wants to send it over decrypted? - if decrypt: - # when decrypting, zfs can't send properties - send_properties = False - else: - # keep data encrypted by sending it raw (including properties) - raw = True - - - (source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, - incompatible_target_snapshots) = \ - self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, - guid_check=guid_check, raw=raw, bookmark_tag=bookmark_tag, - required_snapshots=required_snapshots) - - if show_progress: - self.zfs_node.logger.progress("Pre-cleaning..") - - # NOTE: we do a pre-clean because we dont want filesystems to fillup when backups keep failing. - # Also usefull with no_send to still cleanup stuff. - self._pre_clean( - source_common_snapshot=source_common_snapshot, target_dataset=target_dataset, - target_transfers=target_transfers, target_obsoletes=target_obsoletes, source_obsoletes=source_obsoletes) - - - # check if we can resume - if len(target_transfers)>0: - - if show_progress: - self.zfs_node.logger.progress("Verifying resume token...") - - resume_token = self._validate_resume_token(target_dataset, target_transfers[0]) - else: - resume_token = None - - if show_progress: - self.zfs_node.logger.progress("Preparing...") - - # handle incompatible stuff on target - target_dataset.handle_incompatible_target(incompatible_target_snapshots, destroy_incompatible, force, source_common_snapshot, resume_token is not None) - - # now actually transfer the snapshots, if we want - if no_send or len(target_transfers) == 0: - return - - - (active_filter_properties, active_set_properties) = self.get_allowed_properties(filter_properties, - set_properties) - - # always filter properties that start with the property-format prefix (https://github.com/psy0rz/zfs_autobackup/issues/221) - property_prefix = property_format.split(':')[0] - for prop in self.properties: - if prop.startswith(property_prefix): - active_filter_properties.append(prop) - - # encrypt at target? - if encrypt and not raw: - # filter out encryption properties to let encryption on the target take place - active_filter_properties.extend(["keylocation", "pbkdf2iters", "keyformat", "encryption"]) - write_embedded = False - - # now actually transfer the snapshots - do_rollback = rollback - # When the target dataset is being created fresh and the source is a clone whose - # origin's target equivalent exists, send the first snapshot as an incremental from - # that origin so zfs recv reconstructs the clone relationship on the target. - if source_common_snapshot is None and clone_origin_snapshot is not None: - prev_source_snapshot_bookmark = clone_origin_snapshot - else: - prev_source_snapshot_bookmark = source_common_snapshot - prev_target_snapshot = target_dataset.find_snapshot(source_common_snapshot) - for target_snapshot in target_transfers: - - source_snapshot = self.find_snapshot(target_snapshot) - assert source_snapshot is not None - - # do the rollback, one time at first transfer - if do_rollback: - if show_progress: - self.zfs_node.logger.progress(f"Rolling back {target_dataset}...") - - target_dataset.rollback() - do_rollback = False - - source_snapshot.transfer_snapshot(target_snapshot, features=features, - prev_snapshot=prev_source_snapshot_bookmark, show_progress=show_progress, - filter_properties=active_filter_properties, - set_properties=active_set_properties, - ignore_recv_exit_code=ignore_recv_exit_code, - resume_token=resume_token, write_embedded=write_embedded, raw=raw, - send_properties=send_properties, send_pipes=send_pipes, - recv_pipes=recv_pipes, zfs_compressed=zfs_compressed, force=force) - - resume_token = None - - # hold/release common snapshot on the target. - if holds: - target_snapshot.hold() - - if prev_target_snapshot: - prev_target_snapshot.release() - - if use_bookmarks: - # bookmark common snapshot, and clean up obsolete snapshots and bookmark - source_bookmark = source_snapshot.bookmark(bookmark_tag) - if source_snapshot in source_obsoletes: - source_snapshot.destroy() - - # TODO: make a better is_ours specially for bookmarks, as part of the next refactoring splitting in more classes - # delete any bookmark that ends in ours tag_seprator + tag. - if prev_source_snapshot_bookmark and type( - prev_source_snapshot_bookmark) is ZfsBookmark and prev_source_snapshot_bookmark.name.endswith( - self.zfs_node.tag_seperator + bookmark_tag): - prev_source_snapshot_bookmark.destroy() - - - # dont use bookmarks - else: - source_bookmark = None - if holds: - source_snapshot.hold() - - # release hold, cleanup obsolete snapshot - if isinstance(prev_source_snapshot_bookmark, ZfsSnapshot): - prev_source_snapshot_bookmark.release() - if prev_source_snapshot_bookmark in source_obsoletes: - prev_source_snapshot_bookmark.destroy() - - # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, - # the rest was already destroyed or will not be send) - if prev_target_snapshot is not None and prev_target_snapshot in target_obsoletes: - prev_target_snapshot.destroy() - - # we always try to use the bookmark during incremental send - if source_bookmark: - prev_source_snapshot_bookmark = source_bookmark - else: - prev_source_snapshot_bookmark = source_snapshot - - prev_target_snapshot = target_snapshot - - def handle_incompatible_target(self, incompatible_target_snapshots, destroy_incompatible, force, source_common_snapshot, is_resume): - """destroy incompatbile snapshots on target before sync, or inform user - what to do - - Args: - :type incompatible_target_snapshots: list[ZfsSnapshot] - :type destroy_incompatible: bool - :type force: bool - :type source_common_snapshot: ZfsPointInTime|None - :type is_resume: bool - - """ - - if not self.exists: - # no target yet, so everything ok - return - - if not incompatible_target_snapshots and source_common_snapshot is not None: - # nice existing target with compatible common snapshot. - return - - if not incompatible_target_snapshots and is_resume and len(self.snapshots)==0: - # its a dataset from an incremental send we probably can resume, so its fine - return - - #from this point on things get progressively worse.. - - if source_common_snapshot: - # we just need to delete some incompatibles: - for snapshot in incompatible_target_snapshots: - snapshot.warning("Incompatible snapshot") - - if not destroy_incompatible: - raise (Exception("Use --destroy-incompatible to get rid of these.")) - - else: - # no common snapshot, so the whole dataset is incompatbile - if incompatible_target_snapshots: - if not force or not destroy_incompatible: - self.error("Incompatible dataset!") - raise (Exception("Use --destroy-incompatible -F to overwrite the target dataset and start over.")) - else: - if not force: - self.error("Incompatible dataset!") - raise (Exception("Use -F to overwrite the target dataset and start over.")) - - self.warning("Overwriting incompatible dataset.") - - # remove incompatibles and rollback - for snapshot in incompatible_target_snapshots: - snapshot.destroy(fail_exception=True) - - self.invalidate_cache() - self.rollback() - - def _plan_sync(self, target_dataset, also_other_snapshots, guid_check, raw, bookmark_tag, required_snapshots=None): - """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. - - Args: - :rtype: ( ZfsSnapshot|ZfsBookmark, list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot] ) - :type target_dataset: ZfsContainer - :type also_other_snapshots: bool - :type guid_check: bool - :type raw: bool - :type bookmark_tag: str - :type required_snapshots: set[str]|None - - Returns: - tuple: A tuple containing: - - ZfsSnapshot|ZfsBookmark: The source common snapshot - - list[ZfsSnapshot]: Our obsolete source snapshots, after transfer is done. (will be thinned asap) - - list[ZfsSnapshot]: Our obsolete target snapshots, after transfer is done. (will be thinned asap) - - list[ZfsSnapshot]: Transfer target snapshots. These need to be transferred. - - list[ZfsSnapshot]: Incompatible target snapshots. Target snapshots that are in the way, after the common snapshot. (need to be destroyed to continue) - """ - - ### 1: determine common and start snapshot if target already exists: - - if target_dataset.exists: - target_dataset.debug("Determining start snapshot") - (source_common_snapshot, target_common_snapshot) = self.find_common_snapshot(target_dataset, - guid_check=guid_check, - bookmark_tag=bookmark_tag) - incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(target_common_snapshot, raw) - else: - source_common_snapshot=None - incompatible_target_snapshots =[] - - # let thinner decide whats obsolete on source after the transfer is done - source_obsoletes = [] - if self.our_snapshots: - source_obsoletes = self.thin_list()[1] - - ### 2: Determine possible target snapshots - - # start with snapshots that already exist, minus imcompatibles - if target_dataset.exists: - possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots if - snapshot not in incompatible_target_snapshots] - else: - possible_target_snapshots = [] - - # add all snapshots from the source, starting after the common snapshot if it exists - if source_common_snapshot is not None: - source_snapshot = self.find_next_snapshot(source_common_snapshot) - else: - if self.snapshots: - source_snapshot = self.snapshots[0] - else: - source_snapshot = None - - while source_snapshot: - # we want it? - is_required = required_snapshots is not None and source_snapshot.name in required_snapshots - if is_required and not source_snapshot.is_ours and not also_other_snapshots: - source_snapshot.verbose("Including as clone origin for a selected clone") - if (also_other_snapshots or source_snapshot.is_ours or is_required) and not source_snapshot.is_snapshot_excluded: - # create virtual target snapshot - target_snapshot = target_dataset.zfs_node.get_snapshot( - target_dataset.name + source_snapshot.typed_suffix, force_exists=False) - possible_target_snapshots.append(target_snapshot) - source_snapshot = self.find_next_snapshot(source_snapshot) - - ### 3: Let the thinner decide what it wants by looking at all the possible target_snaphots at once. - # always keep the last target snapshot as common snapshot. - if possible_target_snapshots: - (target_keeps, target_obsoletes) = target_dataset.zfs_node.thin_list(possible_target_snapshots, - keep_snapshots=[ - possible_target_snapshots[-1]]) - else: - target_keeps = [] - target_obsoletes = [] - - ### 4: Look at what the thinner wants and create a list of snapshots we still need to transfer - target_transfers = [] - for target_keep in target_keeps: - if not target_keep.exists: - target_transfers.append(target_keep) - - return source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots - - def _validate_resume_token(self, target_dataset, start_snapshot): - """validate and get (or destory) resume token - - Args: - :type target_dataset: ZfsContainer - :type start_snapshot: ZfsSnapshot|None - :rtype: str|None - """ - - if target_dataset.exists and 'receive_resume_token' in target_dataset.properties: - if start_snapshot is None: - target_dataset.verbose("Aborting resume, its obsolete.") - target_dataset.abort_resume() - else: - resume_token = target_dataset.properties['receive_resume_token'] - # not valid anymore? - resume_snapshot = self.zfs_node.get_resume_snapshot(resume_token) - if not resume_snapshot or start_snapshot.suffix != resume_snapshot.suffix: - target_dataset.verbose("Aborting resume, its no longer valid.") - target_dataset.abort_resume() - else: - return resume_token - - return None diff --git a/zfs_autobackup/sync_datasets.py b/zfs_autobackup/sync_datasets.py index 93e8efb..0a5c37a 100644 --- a/zfs_autobackup/sync_datasets.py +++ b/zfs_autobackup/sync_datasets.py @@ -1,6 +1,7 @@ from .ExecuteNode import ExecuteError from .ThinnerRule import ThinnerRule from .ZfsContainer import ZfsContainer +from .sync_snapshots import sync_snapshots from .util import datetime_now @@ -305,7 +306,7 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta clone_origin_snapshot = _resolve_clone_origin(target_path, strip_path, guid_check, source_dataset, target_node) # sync the snapshots of this dataset - source_dataset.sync_snapshots(target_dataset, show_progress=True, + sync_snapshots(source_dataset, target_dataset, show_progress=True, features=common_features, filter_properties=filter_properties, set_properties=set_properties, ignore_recv_exit_code=ignore_transfer_errors, diff --git a/zfs_autobackup/sync_snapshots.py b/zfs_autobackup/sync_snapshots.py new file mode 100644 index 0000000..564e4d1 --- /dev/null +++ b/zfs_autobackup/sync_snapshots.py @@ -0,0 +1,378 @@ +from .ZfsBookmark import ZfsBookmark +from .ZfsContainer import ZfsContainer +from .ZfsPointInTime import ZfsPointInTime +from .ZfsSnapshot import ZfsSnapshot + + +def _pre_clean(source_dataset, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): + """cleanup old stuff on the source before starting snapshot syncing + + :type source_dataset: ZfsContainer + :type source_common_snapshot: ZfsSnapshot|ZfsBookmark|None + :type target_dataset: ZfsContainer + :type source_obsoletes: list[ZfsSnapshot] + :type target_obsoletes: list[ZfsSnapshot] + :type target_transfers: list[ZfsSnapshot] + """ + + # on source: delete all obsoletes that are not in target_transfers (except common snapshot, if its not a bookmark) + for source_snapshot in source_dataset.snapshots: + if (source_snapshot in source_obsoletes + and source_common_snapshot != source_snapshot + and source_snapshot.find_snapshot_by_suffix(target_transfers) is None): + source_snapshot.destroy() + + # on target: destroy everything thats obsolete, except common_snapshot + if target_dataset.exists: + for target_snapshot in target_dataset.snapshots: + if (target_snapshot in target_obsoletes) \ + and (not source_common_snapshot or ( + target_snapshot.tagless_suffix != source_common_snapshot.tagless_suffix)): + if target_snapshot.exists: + target_snapshot.destroy() + + +def handle_incompatible_target(target_dataset, incompatible_target_snapshots, destroy_incompatible, force, source_common_snapshot, is_resume): + """destroy incompatible snapshots on target before sync, or inform user what to do + + :type target_dataset: ZfsContainer + :type incompatible_target_snapshots: list[ZfsSnapshot] + :type destroy_incompatible: bool + :type force: bool + :type source_common_snapshot: ZfsPointInTime|None + :type is_resume: bool + """ + + if not target_dataset.exists: + # no target yet, so everything ok + return + + if not incompatible_target_snapshots and source_common_snapshot is not None: + # nice existing target with compatible common snapshot. + return + + if not incompatible_target_snapshots and is_resume and len(target_dataset.snapshots) == 0: + # its a dataset from an incremental send we probably can resume, so its fine + return + + #from this point on things get progressively worse.. + + if source_common_snapshot: + # we just need to delete some incompatibles: + for snapshot in incompatible_target_snapshots: + snapshot.warning("Incompatible snapshot") + + if not destroy_incompatible: + raise (Exception("Use --destroy-incompatible to get rid of these.")) + + else: + # no common snapshot, so the whole dataset is incompatible + if incompatible_target_snapshots: + if not force or not destroy_incompatible: + target_dataset.error("Incompatible dataset!") + raise (Exception("Use --destroy-incompatible -F to overwrite the target dataset and start over.")) + else: + if not force: + target_dataset.error("Incompatible dataset!") + raise (Exception("Use -F to overwrite the target dataset and start over.")) + + target_dataset.warning("Overwriting incompatible dataset.") + + # remove incompatibles and rollback + for snapshot in incompatible_target_snapshots: + snapshot.destroy(fail_exception=True) + + target_dataset.invalidate_cache() + target_dataset.rollback() + + +def _validate_resume_token(source_dataset, target_dataset, start_snapshot): + """validate and get (or destroy) resume token + + :type source_dataset: ZfsContainer + :type target_dataset: ZfsContainer + :type start_snapshot: ZfsSnapshot|None + :rtype: str|None + """ + + if target_dataset.exists and 'receive_resume_token' in target_dataset.properties: + if start_snapshot is None: + target_dataset.verbose("Aborting resume, its obsolete.") + target_dataset.abort_resume() + else: + resume_token = target_dataset.properties['receive_resume_token'] + # not valid anymore? + resume_snapshot = source_dataset.zfs_node.get_resume_snapshot(resume_token) + if not resume_snapshot or start_snapshot.suffix != resume_snapshot.suffix: + target_dataset.verbose("Aborting resume, its no longer valid.") + target_dataset.abort_resume() + else: + return resume_token + + return None + + +def _plan_sync(source_dataset, target_dataset, also_other_snapshots, guid_check, raw, bookmark_tag, required_snapshots=None): + """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. + + :type source_dataset: ZfsContainer + :type target_dataset: ZfsContainer + :type also_other_snapshots: bool + :type guid_check: bool + :type raw: bool + :type bookmark_tag: str + :type required_snapshots: set[str]|None + :rtype: ( ZfsSnapshot|ZfsBookmark, list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot] ) + + Returns: + tuple: A tuple containing: + - ZfsSnapshot|ZfsBookmark: The source common snapshot + - list[ZfsSnapshot]: Our obsolete source snapshots, after transfer is done. (will be thinned asap) + - list[ZfsSnapshot]: Our obsolete target snapshots, after transfer is done. (will be thinned asap) + - list[ZfsSnapshot]: Transfer target snapshots. These need to be transferred. + - list[ZfsSnapshot]: Incompatible target snapshots. Target snapshots that are in the way, after the common snapshot. (need to be destroyed to continue) + """ + + ### 1: determine common and start snapshot if target already exists: + + if target_dataset.exists: + target_dataset.debug("Determining start snapshot") + (source_common_snapshot, target_common_snapshot) = source_dataset.find_common_snapshot(target_dataset, + guid_check=guid_check, + bookmark_tag=bookmark_tag) + incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(target_common_snapshot, raw) + else: + source_common_snapshot = None + incompatible_target_snapshots = [] + + # let thinner decide whats obsolete on source after the transfer is done + source_obsoletes = [] + if source_dataset.our_snapshots: + source_obsoletes = source_dataset.thin_list()[1] + + ### 2: Determine possible target snapshots + + # start with snapshots that already exist, minus incompatibles + if target_dataset.exists: + possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots if + snapshot not in incompatible_target_snapshots] + else: + possible_target_snapshots = [] + + # add all snapshots from the source, starting after the common snapshot if it exists + if source_common_snapshot is not None: + source_snapshot = source_dataset.find_next_snapshot(source_common_snapshot) + else: + if source_dataset.snapshots: + source_snapshot = source_dataset.snapshots[0] + else: + source_snapshot = None + + while source_snapshot: + # we want it? + is_required = required_snapshots is not None and source_snapshot.name in required_snapshots + if is_required and not source_snapshot.is_ours and not also_other_snapshots: + source_snapshot.verbose("Including as clone origin for a selected clone") + if (also_other_snapshots or source_snapshot.is_ours or is_required) and not source_snapshot.is_snapshot_excluded: + # create virtual target snapshot + target_snapshot = target_dataset.zfs_node.get_snapshot( + target_dataset.name + source_snapshot.typed_suffix, force_exists=False) + possible_target_snapshots.append(target_snapshot) + source_snapshot = source_dataset.find_next_snapshot(source_snapshot) + + ### 3: Let the thinner decide what it wants by looking at all the possible target_snaphots at once. + # always keep the last target snapshot as common snapshot. + if possible_target_snapshots: + (target_keeps, target_obsoletes) = target_dataset.zfs_node.thin_list(possible_target_snapshots, + keep_snapshots=[ + possible_target_snapshots[-1]]) + else: + target_keeps = [] + target_obsoletes = [] + + ### 4: Look at what the thinner wants and create a list of snapshots we still need to transfer + target_transfers = [] + for target_keep in target_keeps: + if not target_keep.exists: + target_transfers.append(target_keep) + + return source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots + + +def sync_snapshots(source_dataset, target_dataset, features, show_progress, filter_properties, set_properties, + ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, + no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, + use_bookmarks, bookmark_tag, property_format, clone_origin_snapshot=None, + required_snapshots=None): + """sync source_dataset's snapshots to target_dataset, while also thinning out old snapshots along the way. + + :type source_dataset: ZfsContainer + :type target_dataset: ZfsContainer + :type features: list[str] + :type show_progress: bool + :type filter_properties: list[str] + :type set_properties: list[str] + :type ignore_recv_exit_code: bool + :type holds: bool + :type rollback: bool + :type decrypt: bool + :type encrypt: bool + :type also_other_snapshots: bool + :type no_send: bool + :type destroy_incompatible: bool + :type send_pipes: list[str] + :type recv_pipes: list[str] + :type zfs_compressed: bool + :type force: bool + :type guid_check: bool + :type use_bookmarks: bool + :type bookmark_tag: str + :type property_format: str + :type clone_origin_snapshot: ZfsSnapshot|None + :type required_snapshots: set[str]|None + """ + + # defaults for these settings if there is no encryption stuff going on: + send_properties = True + raw = False + write_embedded = True + + # source dataset encrypted? + if source_dataset.properties.get('encryption', 'off') != 'off': + # user wants to send it over decrypted? + if decrypt: + # when decrypting, zfs can't send properties + send_properties = False + else: + # keep data encrypted by sending it raw (including properties) + raw = True + + (source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, + incompatible_target_snapshots) = \ + _plan_sync(source_dataset, target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, + guid_check=guid_check, raw=raw, bookmark_tag=bookmark_tag, + required_snapshots=required_snapshots) + + if show_progress: + source_dataset.zfs_node.logger.progress("Pre-cleaning..") + + # NOTE: we do a pre-clean because we dont want filesystems to fillup when backups keep failing. + # Also usefull with no_send to still cleanup stuff. + _pre_clean( + source_dataset, + source_common_snapshot=source_common_snapshot, target_dataset=target_dataset, + target_transfers=target_transfers, target_obsoletes=target_obsoletes, source_obsoletes=source_obsoletes) + + # check if we can resume + if len(target_transfers) > 0: + + if show_progress: + source_dataset.zfs_node.logger.progress("Verifying resume token...") + + resume_token = _validate_resume_token(source_dataset, target_dataset, target_transfers[0]) + else: + resume_token = None + + if show_progress: + source_dataset.zfs_node.logger.progress("Preparing...") + + # handle incompatible stuff on target + handle_incompatible_target(target_dataset, incompatible_target_snapshots, destroy_incompatible, force, source_common_snapshot, resume_token is not None) + + # now actually transfer the snapshots, if we want + if no_send or len(target_transfers) == 0: + return + + (active_filter_properties, active_set_properties) = source_dataset.get_allowed_properties(filter_properties, + set_properties) + + # always filter properties that start with the property-format prefix (https://github.com/psy0rz/zfs_autobackup/issues/221) + property_prefix = property_format.split(':')[0] + for prop in source_dataset.properties: + if prop.startswith(property_prefix): + active_filter_properties.append(prop) + + # encrypt at target? + if encrypt and not raw: + # filter out encryption properties to let encryption on the target take place + active_filter_properties.extend(["keylocation", "pbkdf2iters", "keyformat", "encryption"]) + write_embedded = False + + # now actually transfer the snapshots + do_rollback = rollback + # When the target dataset is being created fresh and the source is a clone whose + # origin's target equivalent exists, send the first snapshot as an incremental from + # that origin so zfs recv reconstructs the clone relationship on the target. + if source_common_snapshot is None and clone_origin_snapshot is not None: + prev_source_snapshot_bookmark = clone_origin_snapshot + else: + prev_source_snapshot_bookmark = source_common_snapshot + prev_target_snapshot = target_dataset.find_snapshot(source_common_snapshot) + for target_snapshot in target_transfers: + + source_snapshot = source_dataset.find_snapshot(target_snapshot) + assert source_snapshot is not None + + # do the rollback, one time at first transfer + if do_rollback: + if show_progress: + source_dataset.zfs_node.logger.progress(f"Rolling back {target_dataset}...") + + target_dataset.rollback() + do_rollback = False + + source_snapshot.transfer_snapshot(target_snapshot, features=features, + prev_snapshot=prev_source_snapshot_bookmark, show_progress=show_progress, + filter_properties=active_filter_properties, + set_properties=active_set_properties, + ignore_recv_exit_code=ignore_recv_exit_code, + resume_token=resume_token, write_embedded=write_embedded, raw=raw, + send_properties=send_properties, send_pipes=send_pipes, + recv_pipes=recv_pipes, zfs_compressed=zfs_compressed, force=force) + + resume_token = None + + # hold/release common snapshot on the target. + if holds: + target_snapshot.hold() + + if prev_target_snapshot: + prev_target_snapshot.release() + + if use_bookmarks: + # bookmark common snapshot, and clean up obsolete snapshots and bookmark + source_bookmark = source_snapshot.bookmark(bookmark_tag) + if source_snapshot in source_obsoletes: + source_snapshot.destroy() + + # TODO: make a better is_ours specially for bookmarks, as part of the next refactoring splitting in more classes + # delete any bookmark that ends in ours tag_separator + tag. + if prev_source_snapshot_bookmark and type( + prev_source_snapshot_bookmark) is ZfsBookmark and prev_source_snapshot_bookmark.name.endswith( + source_dataset.zfs_node.tag_seperator + bookmark_tag): + prev_source_snapshot_bookmark.destroy() + + # dont use bookmarks + else: + source_bookmark = None + if holds: + source_snapshot.hold() + + # release hold, cleanup obsolete snapshot + if isinstance(prev_source_snapshot_bookmark, ZfsSnapshot): + prev_source_snapshot_bookmark.release() + if prev_source_snapshot_bookmark in source_obsoletes: + prev_source_snapshot_bookmark.destroy() + + # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, + # the rest was already destroyed or will not be send) + if prev_target_snapshot is not None and prev_target_snapshot in target_obsoletes: + prev_target_snapshot.destroy() + + # we always try to use the bookmark during incremental send + if source_bookmark: + prev_source_snapshot_bookmark = source_bookmark + else: + prev_source_snapshot_bookmark = source_snapshot + + prev_target_snapshot = target_snapshot From 06b0bb33cbd094582b85101229c6c485373d2b25 Mon Sep 17 00:00:00 2001 From: DatuX Date: Mon, 8 Jun 2026 10:38:45 +0200 Subject: [PATCH 122/135] refactor --- zfs_autobackup/ZfsContainer.py | 19 ++++--------------- zfs_autobackup/sync_datasets.py | 10 +++++++++- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index 3d06bf4..fd314e8 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -592,10 +592,9 @@ class ZfsContainer(ZfsDataset): self.verbose("Destroying") return super().destroy(fail_exception=fail_exception) - def get_clone_origin_snapshot(self): - """If this dataset is a ZFS clone whose origin can be replicated, return the - source-side origin snapshot. Returns None when the dataset is not a clone, - the origin cannot be parsed, or the clone topology is unreplicatable. + @property + def origin(self): + """The origin snapshot of this clone, or None if this dataset is not a clone or the origin cannot be parsed. :rtype: ZfsSnapshot|None """ @@ -605,17 +604,7 @@ class ZfsContainer(ZfsDataset): return None if '@' not in origin: - self.warning("Cannot replicate as clone: cannot parse origin '{}'.".format(origin)) - return None - - parent_path = origin.split('@', 1)[0] - - # Reverse-clone topology (after 'zfs promote' of a child): origin lives on - # a namespace descendant of this dataset. zfs recv has no way to land a - # clone-creating stream on top of the placeholder that has to exist for - # the descendant, so replication can't preserve the relationship. - if parent_path == self.name or parent_path.startswith(self.name + "/"): - self.warning("Cannot replicate as clone: origin '{}' lives on a namespace descendant of this dataset. Falling back to full send.".format(origin)) + self.warning("Cannot parse clone origin '{}'.".format(origin)) return None return self.zfs_node.get_snapshot(origin) diff --git a/zfs_autobackup/sync_datasets.py b/zfs_autobackup/sync_datasets.py index 0a5c37a..8e738c9 100644 --- a/zfs_autobackup/sync_datasets.py +++ b/zfs_autobackup/sync_datasets.py @@ -107,10 +107,18 @@ def _resolve_clone_origin(target_path, strip_path, guid_check, source_dataset, t :rtype: ZfsSnapshot|None """ - source_origin_snap = source_dataset.get_clone_origin_snapshot() + source_origin_snap = source_dataset.origin if source_origin_snap is None: return None + # Reverse-clone topology (after 'zfs promote' of a child): origin lives on a namespace descendant + # of this dataset. zfs recv cannot land a clone-creating stream on top of the placeholder that has + # to exist for the descendant, so replication can't preserve the relationship. + origin_parent_path = source_origin_snap.name.split('@', 1)[0] + if origin_parent_path == source_dataset.name or origin_parent_path.startswith(source_dataset.name + "/"): + source_dataset.warning("Cannot replicate as clone: origin '{}' lives on a namespace descendant of this dataset. Falling back to full send.".format(source_origin_snap.name)) + return None + try: target_origin_parent = source_origin_snap.parent.map_to_target_path(target_path, strip_path) except Exception as e: From d88572c6cbeca142d25c96bf3b8714dd08c4648e Mon Sep 17 00:00:00 2001 From: DatuX Date: Mon, 8 Jun 2026 11:00:06 +0200 Subject: [PATCH 123/135] refactor --- zfs_autobackup/ZfsDataset.py | 3 ++ zfs_autobackup/sync_datasets.py | 60 +++++++++++++------------------- zfs_autobackup/sync_snapshots.py | 6 ++-- 3 files changed, 30 insertions(+), 39 deletions(-) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 3ca09d0..faadab8 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -87,6 +87,9 @@ class ZfsDataset: return self.name == dataset.name + def __hash__(self): + return hash(self.name) + def verbose(self, txt): """ Args: diff --git a/zfs_autobackup/sync_datasets.py b/zfs_autobackup/sync_datasets.py index 8e738c9..dcee13f 100644 --- a/zfs_autobackup/sync_datasets.py +++ b/zfs_autobackup/sync_datasets.py @@ -154,40 +154,32 @@ def _topological_sort_for_clones(logger, source_datasets): :rtype: list[ZfsContainer] """ - by_name = {d.name: d for d in source_datasets} - - parent_dep = {} # dataset_name -> parent_dataset_name in selection, or None + # build edge map: for each dataset that is a clone of another selected dataset, + # record that dependency so the origin is emitted first. + indegree = {d: 0 for d in source_datasets} + children = {d: [] for d in source_datasets} for d in source_datasets: - try: - origin = d.properties.get('origin', '-') - except Exception: - origin = '-' - if origin == '-' or '@' not in origin: - parent_dep[d.name] = None + origin = d.origin + if origin is None: continue - parent_path = origin.split('@', 1)[0] - if parent_path in by_name and parent_path != d.name: - parent_dep[d.name] = parent_path - else: - parent_dep[d.name] = None + parent = origin.parent + # only a dependency if the origin dataset is itself in the selection + if parent in source_datasets and parent is not d: + indegree[d] += 1 + children[parent].append(d) - indegree = {name: 0 for name in by_name} - children = {name: [] for name in by_name} - for name, parent in parent_dep.items(): - if parent is not None: - indegree[name] += 1 - children[parent].append(name) - - queue = [d.name for d in source_datasets if indegree[d.name] == 0] + # Kahn's algorithm: start with datasets that have no in-selection origin + queue = [d for d in source_datasets if indegree[d] == 0] result = [] while queue: - name = queue.pop(0) - result.append(by_name[name]) - for child in children[name]: + d = queue.pop(0) + result.append(d) + for child in children[d]: indegree[child] -= 1 if indegree[child] == 0: queue.append(child) + # a cycle means ZFS state is inconsistent; fall back to original order if len(result) != len(source_datasets): logger.warning("Clone topological sort: cycle detected, keeping original dataset order.") return source_datasets @@ -201,21 +193,17 @@ def _required_origin_snapshots(source_datasets): force-include those specific snapshots even without --other-snapshots. :type source_datasets: list[ZfsContainer] - :rtype: dict[str, set[str]] + :rtype: dict[ZfsContainer, set[ZfsSnapshot]] """ - by_name = {d.name: d for d in source_datasets} required = {} for d in source_datasets: - try: - origin = d.properties.get('origin', '-') - except Exception: - origin = '-' - if origin == '-' or '@' not in origin: + origin = d.origin + if origin is None: continue - parent_path = origin.split('@', 1)[0] - if parent_path in by_name and parent_path != d.name: - required.setdefault(parent_path, set()).add(origin) + parent = origin.parent + if parent in source_datasets and parent is not d: + required.setdefault(parent, set()).add(origin) return required @@ -329,7 +317,7 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta bookmark_tag=bookmark_tag, property_format=property_format, clone_origin_snapshot=clone_origin_snapshot, - required_snapshots=required_origin_snapshots.get(source_dataset.name)) + required_snapshots=required_origin_snapshots.get(source_dataset)) except Exception as e: fail_count = fail_count + 1 diff --git a/zfs_autobackup/sync_snapshots.py b/zfs_autobackup/sync_snapshots.py index 564e4d1..d0ff4eb 100644 --- a/zfs_autobackup/sync_snapshots.py +++ b/zfs_autobackup/sync_snapshots.py @@ -121,7 +121,7 @@ def _plan_sync(source_dataset, target_dataset, also_other_snapshots, guid_check, :type guid_check: bool :type raw: bool :type bookmark_tag: str - :type required_snapshots: set[str]|None + :type required_snapshots: set[ZfsSnapshot]|None :rtype: ( ZfsSnapshot|ZfsBookmark, list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot] ) Returns: @@ -170,7 +170,7 @@ def _plan_sync(source_dataset, target_dataset, also_other_snapshots, guid_check, while source_snapshot: # we want it? - is_required = required_snapshots is not None and source_snapshot.name in required_snapshots + is_required = required_snapshots is not None and source_snapshot in required_snapshots if is_required and not source_snapshot.is_ours and not also_other_snapshots: source_snapshot.verbose("Including as clone origin for a selected clone") if (also_other_snapshots or source_snapshot.is_ours or is_required) and not source_snapshot.is_snapshot_excluded: @@ -229,7 +229,7 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt :type bookmark_tag: str :type property_format: str :type clone_origin_snapshot: ZfsSnapshot|None - :type required_snapshots: set[str]|None + :type required_snapshots: set[ZfsSnapshot]|None """ # defaults for these settings if there is no encryption stuff going on: From 431417c82c9a73e6f37eeb7c68a467018dbae9c6 Mon Sep 17 00:00:00 2001 From: DatuX Date: Mon, 8 Jun 2026 11:31:14 +0200 Subject: [PATCH 124/135] fix --- zfs_autobackup/ZfsNode.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index f516b7f..8fcdc83 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -223,11 +223,12 @@ class ZfsNode(ExecuteNode): self._progress_start_time = time.time() elif progress_fields[1].isnumeric(): bytes_ = int(progress_fields[1]) - if self._progress_total_bytes: + elapsed = time.time() - self._progress_start_time + if self._progress_total_bytes and elapsed > 0 and bytes_ > 0: percentage = min(100, int(bytes_ * 100 / self._progress_total_bytes)) - speed = int(bytes_ / (time.time() - self._progress_start_time) / (1024 * 1024)) + speed = int(bytes_ / elapsed / (1024 * 1024)) bytes_left = self._progress_total_bytes - bytes_ - minutes_left = int((bytes_left / (bytes_ / (time.time() - self._progress_start_time))) / 60) + minutes_left = int(bytes_left / (bytes_ / elapsed) / 60) self.logger.progress( "Transfering {}% {}MB/s (total {}MB, {} minutes left)".format(percentage, speed, int( From 4106fb8c7213ece9756205ce3f2ea1864ea60661 Mon Sep 17 00:00:00 2001 From: DatuX Date: Mon, 8 Jun 2026 12:43:57 +0200 Subject: [PATCH 125/135] add --test tests --- tests/test_clone.py | 135 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 121 insertions(+), 14 deletions(-) diff --git a/tests/test_clone.py b/tests/test_clone.py index 574b302..cd73450 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -19,6 +19,20 @@ class TestZfsAutobackupClone(unittest2.TestCase): shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( "test test_target1 --no-progress --verbose".split(" ")).run()) @@ -65,15 +79,31 @@ test_target1/test_source2/fs2/sub@test-20101111000000 shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") shelltest("zfs set autobackup:test=true test_source1/fs1_clone") - with OutputIO() as buf: - with redirect_stdout(buf), redirect_stderr(buf): + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertIn("Cannot replicate as clone", combined) + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( "test test_target1 --no-progress --verbose".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() - output = buf.getvalue() - print(output) - self.assertIn("Cannot replicate as clone", output) + print(combined) + self.assertIn("Cannot replicate as clone", combined) + self.assertNotIn("STDERR", combined) origin = shelltest("zfs get -H -o value origin test_target1/test_source1/fs1_clone").strip() self.assertEqual(origin, "-") @@ -88,6 +118,20 @@ test_target1/test_source2/fs2/sub@test-20101111000000 shelltest("zfs clone test_source1/fs1@test-20101111000000 test_source1/fs1_clone") shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) @@ -102,6 +146,20 @@ test_target1/test_source2/fs2/sub@test-20101111000000 shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-clone --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( "test test_target1 --no-progress --verbose --no-clone".split(" ")).run()) @@ -120,6 +178,20 @@ test_target1/test_source2/fs2/sub@test-20101111000000 self.assertFalse(ZfsAutobackup( "test test_target1 --no-progress --verbose".split(" ")).run()) + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) @@ -141,6 +213,20 @@ test_target1/test_source1/fs1_clone@test-20101111000001 shelltest("zfs clone test_source1/fs1@base test_source1/aaa_clone") shelltest("zfs set autobackup:test=true test_source1/aaa_clone") + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( "test test_target1 --no-progress --verbose".split(" ")).run()) @@ -155,9 +241,15 @@ test_target1/test_source1/fs1_clone@test-20101111000001 shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") shelltest("zfs set autobackup:test=true test_source1/fs1_clone") - with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --test".split(" ")).run()) + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertNotIn("STDERR", combined) r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ @@ -179,14 +271,29 @@ test_target1 origin = shelltest("zfs get -H -o value origin test_source1/fs1").strip() self.assertEqual(origin, "test_source1/fs1/inner_clone@s1") - with OutputIO() as buf: - with redirect_stdout(buf), redirect_stderr(buf): + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + result = ZfsAutobackup( + "test test_target1 --no-progress --verbose --test".split(" ")).run() + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertTrue(result) + self.assertIn("lives on a namespace descendant", combined) + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): with mocktime("20101111000000"): result = ZfsAutobackup( "test test_target1 --no-progress --verbose".split(" ")).run() + combined = buf_out.getvalue() + buf_err.getvalue() - output = buf.getvalue() - print(output) - + print(combined) self.assertTrue(result) - self.assertIn("lives on a namespace descendant", output) + self.assertIn("lives on a namespace descendant", combined) From c2595cb2d0abea0b8a9e3410b745fc2c95098292 Mon Sep 17 00:00:00 2001 From: DatuX Date: Wed, 10 Jun 2026 09:29:09 +0200 Subject: [PATCH 126/135] fixes --- tests/test_clone.py | 9 +++-- zfs_autobackup/ZfsContainer.py | 15 +++++++- zfs_autobackup/ZfsDataset.py | 31 +++++++++++++--- zfs_autobackup/ZfsPointInTime.py | 5 --- zfs_autobackup/ZfsSnapshot.py | 3 +- zfs_autobackup/sync_datasets.py | 56 +--------------------------- zfs_autobackup/sync_snapshots.py | 63 +++++++++++++++++++++++++++++--- 7 files changed, 107 insertions(+), 75 deletions(-) diff --git a/tests/test_clone.py b/tests/test_clone.py index cd73450..7ee94ea 100644 --- a/tests/test_clone.py +++ b/tests/test_clone.py @@ -29,6 +29,7 @@ class TestZfsAutobackupClone(unittest2.TestCase): combined = buf_out.getvalue() + buf_err.getvalue() print(combined) + self.assertNotIn("guid", combined) # testing can give guid errors self.assertNotIn("STDERR", combined) after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) self.assertMultiLineEqual(before, after) @@ -277,11 +278,11 @@ test_target1 with redirect_stdout(buf_out), redirect_stderr(buf_err): with mocktime("20101111000000"): result = ZfsAutobackup( - "test test_target1 --no-progress --verbose --test".split(" ")).run() + "test test_target1 --no-progress --verbose --test -F".split(" ")).run() combined = buf_out.getvalue() + buf_err.getvalue() print(combined) - self.assertTrue(result) + self.assertFalse(result) self.assertIn("lives on a namespace descendant", combined) self.assertNotIn("STDERR", combined) after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) @@ -291,9 +292,9 @@ test_target1 with redirect_stdout(buf_out), redirect_stderr(buf_err): with mocktime("20101111000000"): result = ZfsAutobackup( - "test test_target1 --no-progress --verbose".split(" ")).run() + "test test_target1 --no-progress --verbose -F".split(" ")).run() combined = buf_out.getvalue() + buf_err.getvalue() print(combined) - self.assertTrue(result) + self.assertFalse(result) self.assertIn("lives on a namespace descendant", combined) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index fd314e8..b2cb108 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -21,6 +21,13 @@ class ZfsContainer(ZfsDataset): self._datasets = None # type: None|list[ZfsContainer] self._snapshots_bookmarks = None # type: None|list[ZfsSnapshot|ZfsBookmark] + + # def __str__(self): + # + # return f"{self.zfs_node}: {self.name} {'force_exists' if self.force_exists else ''} {len(self._snapshots_bookmarks) if self._snapshots_bookmarks is not None else ''}" + + + @property def parent(self): """get parent dataset @@ -266,6 +273,8 @@ class ZfsContainer(ZfsDataset): def rollback(self): """rollback to latest existing snapshot on this dataset""" + print (f"HIJ IS {self._snapshots_bookmarks}") + for snapshot in reversed(self.snapshots): if snapshot.exists: snapshot.rollback() @@ -375,6 +384,10 @@ class ZfsContainer(ZfsDataset): self.force_exists = True + if self.zfs_node.readonly: + self._snapshots_bookmarks=[] + self.simulate_properties() + def cache_snapshot_bookmark(self, snapshot, force=False): """Update our snapshot and bookmark cache (if we have any). Use force if you want to force the caching, potentially triggering a zfs list Args: @@ -509,7 +522,7 @@ class ZfsContainer(ZfsDataset): # Source snapshot with same suffix? source_snapshot = self.find_snapshot(target_snapshot) if source_snapshot: - if guid_check and source_snapshot.properties['guid'] != target_snapshot.properties['guid']: + if guid_check and not source_snapshot.guid_matches(target_snapshot): source_snapshot.warning("Snapshot has mismatching GUID, ignoring.") else: source_snapshot.debug("Common snapshot") diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index faadab8..7ac21eb 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -77,6 +77,7 @@ class ZfsDataset: def __str__(self): + # return f"{self.zfs_node}: {self.name} {'force_exists' if self.force_exists else ''}" return self.name def __eq__(self, dataset): @@ -90,33 +91,46 @@ class ZfsDataset: def __hash__(self): return hash(self.name) + def guid_matches(self, other): + """Compare GUIDs between this dataset and other. Returns True if they match, + or if either side has an empty guid (test mode simulation — can't compare). + + :type other: ZfsDataset + :rtype: bool + """ + guid_self = self.properties.get('guid', '') + guid_other = other.properties.get('guid', '') + if guid_self == '' or guid_other == '': + return True + return guid_self == guid_other + def verbose(self, txt): """ Args: :type txt: str """ - self.zfs_node.verbose("{}: {}".format(self.name, txt)) + self.zfs_node.verbose("{}: {}".format(self, txt)) def error(self, txt): """ Args: :type txt: str """ - self.zfs_node.error("{}: {}".format(self.name, txt)) + self.zfs_node.error("{}: {}".format(self, txt)) def warning(self, txt): """ Args: :type txt: str """ - self.zfs_node.warning("{}: {}".format(self.name, txt)) + self.zfs_node.warning("{}: {}".format(self, txt)) def debug(self, txt): """ Args: :type txt: str """ - self.zfs_node.debug("{}: {}".format(self.name, txt)) + self.zfs_node.debug("{}: {}".format(self, txt)) def split_path(self): """return the path elements as an array""" @@ -207,8 +221,8 @@ class ZfsDataset: ] self.debug("Getting zfs properties") - properties = {} # type: dict[str, str] + output = cast("list[list[str]]", cast(object, self.zfs_node.run( tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]))) for pair in output: @@ -289,3 +303,10 @@ class ZfsDataset: ] self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) + + def simulate_properties(self): + """in test mode when we create fake snapshots or bookmarks, we also need to simluate properties""" + if self._properties is None: + self._properties={ + 'guid': '' + } diff --git a/zfs_autobackup/ZfsPointInTime.py b/zfs_autobackup/ZfsPointInTime.py index 7325016..43e5985 100644 --- a/zfs_autobackup/ZfsPointInTime.py +++ b/zfs_autobackup/ZfsPointInTime.py @@ -101,8 +101,3 @@ class ZfsPointInTime(ZfsDataset): return None - def simulate_properties(self): - """in test mode when we create fake snapshots or bookmarks, we also need to simluate properties""" - self._properties={ - 'guid': self.tagless_suffix # just use the snapshot name as "guid" - } diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py index 1c7e700..df1e40c 100644 --- a/zfs_autobackup/ZfsSnapshot.py +++ b/zfs_autobackup/ZfsSnapshot.py @@ -183,6 +183,7 @@ class ZfsSnapshot(ZfsPointInTime): # in test mode we pretend everything was ok and it exists if self.zfs_node.readonly: self.force_exists = True + self.simulate_properties() # check if transfer was really ok (exit codes have been wrong before due to bugs in zfs-utils and some # errors should be ignored, thats where the ignore_exitcodes is for.) @@ -249,7 +250,7 @@ class ZfsSnapshot(ZfsPointInTime): # try to automount it, if its the initial transfer if not prev_snapshot: - # in test mode it doesnt actually exist, so dont try to mount it/read properties + # in test mode it doesnt actually exist, so dont try to mount it/read properties, and simulate properties. if not target_snapshot.zfs_node.readonly: target_snapshot.parent.automount() diff --git a/zfs_autobackup/sync_datasets.py b/zfs_autobackup/sync_datasets.py index dcee13f..fffba9b 100644 --- a/zfs_autobackup/sync_datasets.py +++ b/zfs_autobackup/sync_datasets.py @@ -1,4 +1,3 @@ -from .ExecuteNode import ExecuteError from .ThinnerRule import ThinnerRule from .ZfsContainer import ZfsContainer from .sync_snapshots import sync_snapshots @@ -95,54 +94,6 @@ def destroy_missing_targets(logger, target_dataset, used_target_datasets, destro dataset.error("Error during --destroy-missing: {}".format(str(e))) -def _resolve_clone_origin(target_path, strip_path, guid_check, source_dataset, target_node): - """Return the source-side origin snapshot to use as zfs send -i base for a clone, - or None if the clone relationship cannot be preserved on the target. - - :type target_path: str - :type strip_path: int - :type guid_check: bool - :type source_dataset: ZfsContainer - :type target_node: ZfsNode - :rtype: ZfsSnapshot|None - """ - - source_origin_snap = source_dataset.origin - if source_origin_snap is None: - return None - - # Reverse-clone topology (after 'zfs promote' of a child): origin lives on a namespace descendant - # of this dataset. zfs recv cannot land a clone-creating stream on top of the placeholder that has - # to exist for the descendant, so replication can't preserve the relationship. - origin_parent_path = source_origin_snap.name.split('@', 1)[0] - if origin_parent_path == source_dataset.name or origin_parent_path.startswith(source_dataset.name + "/"): - source_dataset.warning("Cannot replicate as clone: origin '{}' lives on a namespace descendant of this dataset. Falling back to full send.".format(source_origin_snap.name)) - return None - - try: - target_origin_parent = source_origin_snap.parent.map_to_target_path(target_path, strip_path) - except Exception as e: - source_dataset.warning("Cannot replicate as clone: cannot map origin '{}' to target ({}). Falling back to full send.".format(source_origin_snap.name, str(e))) - return None - - target_origin_snap = target_node.get_snapshot(target_origin_parent + '@' + source_origin_snap.suffix) - - if not target_origin_snap.exists: - source_dataset.warning("Cannot replicate as clone: origin '{}' not available on target. Falling back to full send.".format(source_origin_snap.name)) - return None - - if guid_check: - try: - if source_origin_snap.properties.get('guid') != target_origin_snap.properties.get('guid'): - source_dataset.warning("Cannot replicate as clone: origin guid mismatch between source and target. Falling back to full send.") - return None - except ExecuteError: - # properties not readable (e.g. test mode after a simulated recv) — skip the check - source_dataset.debug("Cannot read origin guid; skipping guid check for clone replication.") - - return source_origin_snap - - def _topological_sort_for_clones(logger, source_datasets): """Reorder source_datasets so an origin is processed before any selected clone of it. Datasets whose origin is not in the selection (or who are not clones) are independent @@ -297,10 +248,6 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta # if the source is a clone and the target-side origin is available, send # the first snapshot as an incremental from the origin so the clone # relationship is preserved on the target. - clone_origin_snapshot = None - if not no_clone: - clone_origin_snapshot = _resolve_clone_origin(target_path, strip_path, guid_check, source_dataset, target_node) - # sync the snapshots of this dataset sync_snapshots(source_dataset, target_dataset, show_progress=True, features=common_features, filter_properties=filter_properties, @@ -316,7 +263,8 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta guid_check=guid_check, use_bookmarks=use_bookmarks, bookmark_tag=bookmark_tag, property_format=property_format, - clone_origin_snapshot=clone_origin_snapshot, + no_clone=no_clone, + target_path=target_path, strip_path=strip_path, required_snapshots=required_origin_snapshots.get(source_dataset)) except Exception as e: diff --git a/zfs_autobackup/sync_snapshots.py b/zfs_autobackup/sync_snapshots.py index d0ff4eb..366be2f 100644 --- a/zfs_autobackup/sync_snapshots.py +++ b/zfs_autobackup/sync_snapshots.py @@ -1,9 +1,54 @@ +from .ExecuteNode import ExecuteError from .ZfsBookmark import ZfsBookmark from .ZfsContainer import ZfsContainer from .ZfsPointInTime import ZfsPointInTime from .ZfsSnapshot import ZfsSnapshot +def _resolve_clone_origin(target_path, strip_path, guid_check, source_dataset, target_node): + """Return the source-side origin snapshot to use as zfs send -i base for a clone, + or None if the clone relationship cannot be preserved on the target. + + :type target_path: str + :type strip_path: int + :type guid_check: bool + :type source_dataset: ZfsContainer + :type target_node: ZfsNode + :rtype: ZfsSnapshot|None + """ + + source_origin_snap = source_dataset.origin + if source_origin_snap is None: + return None + + # Reverse-clone topology (after 'zfs promote' of a child): origin lives on a namespace descendant + # of this dataset. zfs recv cannot land a clone-creating stream on top of the placeholder that has + # to exist for the descendant, so replication can't preserve the relationship. + origin_parent_path = source_origin_snap.name.split('@', 1)[0] + if origin_parent_path == source_dataset.name or origin_parent_path.startswith(source_dataset.name + "/"): + source_dataset.warning("Cannot replicate as clone: origin '{}' lives on a namespace descendant of this dataset. Falling back to full send.".format(source_origin_snap.name)) + return None + + try: + target_origin_parent = source_origin_snap.parent.map_to_target_path(target_path, strip_path) + except Exception as e: + source_dataset.warning("Cannot replicate as clone: cannot map origin '{}' to target ({}). Falling back to full send.".format(source_origin_snap.name, str(e))) + return None + + target_origin_snap = target_node.get_snapshot(target_origin_parent + '@' + source_origin_snap.suffix) + + if not target_origin_snap.exists: + source_dataset.warning("Cannot replicate as clone: origin '{}' not available on target. Falling back to full send.".format(source_origin_snap.name)) + return None + + if guid_check: + if not source_origin_snap.guid_matches(target_origin_snap): + source_dataset.warning("Cannot replicate as clone: origin guid mismatch between source and target. Falling back to full send.") + return None + + return source_origin_snap + + def _pre_clean(source_dataset, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): """cleanup old stuff on the source before starting snapshot syncing @@ -82,7 +127,7 @@ def handle_incompatible_target(target_dataset, incompatible_target_snapshots, de for snapshot in incompatible_target_snapshots: snapshot.destroy(fail_exception=True) - target_dataset.invalidate_cache() + # target_dataset.invalidate_cache() target_dataset.rollback() @@ -202,8 +247,8 @@ def _plan_sync(source_dataset, target_dataset, also_other_snapshots, guid_check, def sync_snapshots(source_dataset, target_dataset, features, show_progress, filter_properties, set_properties, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, - use_bookmarks, bookmark_tag, property_format, clone_origin_snapshot=None, - required_snapshots=None): + use_bookmarks, bookmark_tag, property_format, no_clone=True, + target_path=None, strip_path=0, required_snapshots=None): """sync source_dataset's snapshots to target_dataset, while also thinning out old snapshots along the way. :type source_dataset: ZfsContainer @@ -228,7 +273,9 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt :type use_bookmarks: bool :type bookmark_tag: str :type property_format: str - :type clone_origin_snapshot: ZfsSnapshot|None + :type no_clone: bool + :type target_path: str + :type strip_path: int :type required_snapshots: set[ZfsSnapshot]|None """ @@ -303,7 +350,13 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt # When the target dataset is being created fresh and the source is a clone whose # origin's target equivalent exists, send the first snapshot as an incremental from # that origin so zfs recv reconstructs the clone relationship on the target. - if source_common_snapshot is None and clone_origin_snapshot is not None: + # Only resolve this now (not earlier) — checking target origin existence is only + # valid when we know this is actually a new full transfer. + if source_common_snapshot is None and not no_clone: + clone_origin_snapshot = _resolve_clone_origin(target_path, strip_path, guid_check, source_dataset, target_dataset.zfs_node) + else: + clone_origin_snapshot = None + if clone_origin_snapshot is not None: prev_source_snapshot_bookmark = clone_origin_snapshot else: prev_source_snapshot_bookmark = source_common_snapshot From 8abc07eea2c9ee0260011ac8ad3d4f907e007a75 Mon Sep 17 00:00:00 2001 From: DatuX Date: Wed, 10 Jun 2026 09:57:40 +0200 Subject: [PATCH 127/135] refactor --- zfs_autobackup/sync_datasets.py | 241 +++++++++++++------------ zfs_autobackup/sync_snapshots.py | 291 ++++++++++++++++++------------- 2 files changed, 293 insertions(+), 239 deletions(-) diff --git a/zfs_autobackup/sync_datasets.py b/zfs_autobackup/sync_datasets.py index fffba9b..ce69601 100644 --- a/zfs_autobackup/sync_datasets.py +++ b/zfs_autobackup/sync_datasets.py @@ -1,99 +1,130 @@ +"""Dataset-level replication: sync a selection of source datasets to a target node, +and thin or destroy target datasets that are missing on the source.""" + +from collections import deque + from .ThinnerRule import ThinnerRule from .ZfsContainer import ZfsContainer from .sync_snapshots import sync_snapshots from .util import datetime_now +def _missing_datasets(target_dataset, used_target_datasets): + """list the datasets under target_dataset that are not used as a backup target (anymore) + + :type target_dataset: ZfsContainer + :type used_target_datasets: list[ZfsContainer] + :rtype: list[ZfsContainer] + """ + + return [dataset for dataset in target_dataset.recursive_datasets if dataset not in used_target_datasets] + + def thin_missing_targets(logger, target_dataset, used_target_datasets): """thin target datasets that are missing on the source. + :type logger: LogConsole - :type used_target_datasets: list[ZfsContainer] :type target_dataset: ZfsContainer + :type used_target_datasets: list[ZfsContainer] """ logger.debug("Thinning obsolete datasets") - missing_datasets = [dataset for dataset in target_dataset.recursive_datasets if - dataset not in used_target_datasets] + missing_datasets = _missing_datasets(target_dataset, used_target_datasets) - count = 0 - for dataset in missing_datasets: + for count, dataset in enumerate(missing_datasets, start=1): logger.debug("analyse missing {}".format(dataset)) - - count = count + 1 logger.progress("Analysing missing..", count, len(missing_datasets), 0) try: dataset.debug("Missing on source, thinning") dataset.thin() - except Exception as e: dataset.error("Error during thinning of missing datasets ({})".format(str(e))) +def _destroy_missing_dataset(dataset, destroy_missing, utc): + """destroy a single missing dataset if it meets all the requirements, otherwise tell the user why not. + + :type dataset: ZfsContainer + :type destroy_missing: str + :type utc: bool + """ + + # cant do anything without our own snapshots + if not dataset.our_snapshots: + if dataset.datasets: + # its not a leaf, just ignore + dataset.debug("Destroy missing: ignoring") + else: + dataset.verbose("Destroy missing: has no snapshots made by us (please destroy manually).") + return + + # past the deadline? + deadline_ttl = ThinnerRule("0s" + destroy_missing).ttl + now = datetime_now(utc).timestamp() + if dataset.our_snapshots[-1].timestamp + deadline_ttl > now: + dataset.verbose("Destroy missing: Waiting for deadline.") + return + + dataset.debug("Destroy missing: Removing our snapshots.") + + # remove all our snaphots, except last, to safe space in case we fail later on + for snapshot in dataset.our_snapshots[:-1]: + snapshot.destroy(fail_exception=True) + + # does it have other snapshots? + if any(not snapshot.is_ours for snapshot in dataset.snapshots): + dataset.verbose("Destroy missing: Still in use by other snapshots") + return + + if dataset.datasets: + dataset.verbose("Destroy missing: Still has children here.") + return + + dataset.verbose("Destroy missing.") + dataset.our_snapshots[-1].destroy(fail_exception=True) + dataset.destroy(fail_exception=True) + + def destroy_missing_targets(logger, target_dataset, used_target_datasets, destroy_missing, utc): """destroy target datasets that are missing on the source and that meet the requirements + :type logger: LogConsole - :type used_target_datasets: list[ZfsContainer] :type target_dataset: ZfsContainer + :type used_target_datasets: list[ZfsContainer] :type destroy_missing: str :type utc: bool """ logger.debug("Destroying obsolete datasets") + missing_datasets = _missing_datasets(target_dataset, used_target_datasets) - missing_datasets = [dataset for dataset in target_dataset.recursive_datasets if - dataset not in used_target_datasets] - - count = 0 - for dataset in missing_datasets: - - count = count + 1 + for count, dataset in enumerate(missing_datasets, start=1): logger.progress("Analysing destroy missing...", count, len(missing_datasets), 0) try: - # cant do anything without our own snapshots - if not dataset.our_snapshots: - if dataset.datasets: - # its not a leaf, just ignore - dataset.debug("Destroy missing: ignoring") - else: - dataset.verbose( - "Destroy missing: has no snapshots made by us (please destroy manually).") - else: - # past the deadline? - deadline_ttl = ThinnerRule("0s" + destroy_missing).ttl - now = datetime_now(utc).timestamp() - if dataset.our_snapshots[-1].timestamp + deadline_ttl > now: - dataset.verbose("Destroy missing: Waiting for deadline.") - else: - - dataset.debug("Destroy missing: Removing our snapshots.") - - # remove all our snaphots, except last, to safe space in case we fail later on - for snapshot in dataset.our_snapshots[:-1]: - snapshot.destroy(fail_exception=True) - - # does it have other snapshots? - has_others = False - for snapshot in dataset.snapshots: - if not snapshot.is_ours: - has_others = True - break - - if has_others: - dataset.verbose("Destroy missing: Still in use by other snapshots") - else: - if dataset.datasets: - dataset.verbose("Destroy missing: Still has children here.") - else: - dataset.verbose("Destroy missing.") - dataset.our_snapshots[-1].destroy(fail_exception=True) - dataset.destroy(fail_exception=True) - + _destroy_missing_dataset(dataset, destroy_missing, utc) except Exception as e: dataset.error("Error during --destroy-missing: {}".format(str(e))) +def _clone_dependencies(source_datasets): + """yield (clone_dataset, origin_snapshot, origin_dataset) for every selected dataset that is a + clone of another selected dataset. + + :type source_datasets: list[ZfsContainer] + """ + + for dataset in source_datasets: + origin_snapshot = dataset.origin + if origin_snapshot is None: + continue + origin_dataset = origin_snapshot.parent + # only a dependency if the origin dataset is itself in the selection + if origin_dataset in source_datasets and origin_dataset is not dataset: + yield dataset, origin_snapshot, origin_dataset + + def _topological_sort_for_clones(logger, source_datasets): """Reorder source_datasets so an origin is processed before any selected clone of it. Datasets whose origin is not in the selection (or who are not clones) are independent @@ -107,25 +138,19 @@ def _topological_sort_for_clones(logger, source_datasets): # build edge map: for each dataset that is a clone of another selected dataset, # record that dependency so the origin is emitted first. - indegree = {d: 0 for d in source_datasets} - children = {d: [] for d in source_datasets} - for d in source_datasets: - origin = d.origin - if origin is None: - continue - parent = origin.parent - # only a dependency if the origin dataset is itself in the selection - if parent in source_datasets and parent is not d: - indegree[d] += 1 - children[parent].append(d) + indegree = {dataset: 0 for dataset in source_datasets} + children = {dataset: [] for dataset in source_datasets} + for clone_dataset, origin_snapshot, origin_dataset in _clone_dependencies(source_datasets): + indegree[clone_dataset] += 1 + children[origin_dataset].append(clone_dataset) # Kahn's algorithm: start with datasets that have no in-selection origin - queue = [d for d in source_datasets if indegree[d] == 0] + queue = deque(dataset for dataset in source_datasets if indegree[dataset] == 0) result = [] while queue: - d = queue.pop(0) - result.append(d) - for child in children[d]: + dataset = queue.popleft() + result.append(dataset) + for child in children[dataset]: indegree[child] -= 1 if indegree[child] == 0: queue.append(child) @@ -139,22 +164,17 @@ def _topological_sort_for_clones(logger, source_datasets): def _required_origin_snapshots(source_datasets): - """Build a map parent_dataset_name -> set of origin snapshot full names that - downstream clones (also in the selection) need pinned on the target. Used to - force-include those specific snapshots even without --other-snapshots. + """Build a map origin_dataset -> set of origin snapshots that downstream clones (also in the + selection) need pinned on the target. Used to force-include those specific snapshots even + without --other-snapshots. :type source_datasets: list[ZfsContainer] :rtype: dict[ZfsContainer, set[ZfsSnapshot]] """ required = {} - for d in source_datasets: - origin = d.origin - if origin is None: - continue - parent = origin.parent - if parent in source_datasets and parent is not d: - required.setdefault(parent, set()).add(origin) + for clone_dataset, origin_snapshot, origin_dataset in _clone_dependencies(source_datasets): + required.setdefault(origin_dataset, set()).add(origin_snapshot) return required @@ -167,13 +187,14 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta guid_check, property_format, debug, target_dataset_base, destroy_missing, utc): """Sync datasets, or thin-only on both sides. + :type logger: LogConsole :type source_node: ZfsNode :type source_datasets: list[ZfsContainer] :type target_node: ZfsNode :type bookmark_tag: str - :type send_pipes: list - :type recv_pipes: list + :type send_pipes: list[str] + :type recv_pipes: list[str] :type target_path: str :type strip_path: int :type no_clone: bool @@ -194,7 +215,7 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta :type guid_check: bool :type property_format: str :type debug: bool - :type target_dataset: ZfsContainer + :type target_dataset_base: ZfsContainer :type destroy_missing: str|None :type utc: bool :rtype: int @@ -207,11 +228,9 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta required_origin_snapshots = {} fail_count = 0 - count = 0 target_datasets = [] - for source_dataset in source_datasets: + for count, source_dataset in enumerate(source_datasets, start=1): - count = count + 1 logger.progress("Analysing dataset...", count, len(source_datasets), fail_count) try: @@ -233,41 +252,33 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta # determine common zpool features (cached, so no problem we call it often) source_features = source_node.get_pool(source_dataset).features target_features = target_node.get_pool(target_dataset).features - common_features = [f for f in source_features if f in target_features] + common_features = [feature for feature in source_features if feature in target_features] - if no_bookmarks: + # NOTE: bookmark_written seems to be needed. (only 'bookmarks' was not enough on ubuntu 20) + use_bookmarks = not no_bookmarks + if use_bookmarks and 'bookmark_written' not in common_features: + source_dataset.warning("Disabling bookmarks, not supported on both pools.") use_bookmarks = False - else: - # NOTE: bookmark_written seems to be needed. (only 'bookmarks' was not enough on ubuntu 20) - if 'bookmark_written' not in common_features: - source_dataset.warning("Disabling bookmarks, not supported on both pools.") - use_bookmarks = False - else: - use_bookmarks = True - # if the source is a clone and the target-side origin is available, send - # the first snapshot as an incremental from the origin so the clone - # relationship is preserved on the target. # sync the snapshots of this dataset sync_snapshots(source_dataset, target_dataset, show_progress=True, - features=common_features, filter_properties=filter_properties, - set_properties=set_properties, - ignore_recv_exit_code=ignore_transfer_errors, - holds=holds, rollback=rollback, - also_other_snapshots=other_snapshots, - no_send=no_send, - destroy_incompatible=destroy_incompatible, - send_pipes=send_pipes, recv_pipes=recv_pipes, - decrypt=decrypt, encrypt=encrypt, - zfs_compressed=zfs_compressed, force=force, - guid_check=guid_check, use_bookmarks=use_bookmarks, - bookmark_tag=bookmark_tag, - property_format=property_format, - no_clone=no_clone, - target_path=target_path, strip_path=strip_path, - required_snapshots=required_origin_snapshots.get(source_dataset)) + features=common_features, filter_properties=filter_properties, + set_properties=set_properties, + ignore_recv_exit_code=ignore_transfer_errors, + holds=holds, rollback=rollback, + also_other_snapshots=other_snapshots, + no_send=no_send, + destroy_incompatible=destroy_incompatible, + send_pipes=send_pipes, recv_pipes=recv_pipes, + decrypt=decrypt, encrypt=encrypt, + zfs_compressed=zfs_compressed, force=force, + guid_check=guid_check, use_bookmarks=use_bookmarks, + bookmark_tag=bookmark_tag, + property_format=property_format, + no_clone=no_clone, + target_path=target_path, strip_path=strip_path, + required_snapshots=required_origin_snapshots.get(source_dataset)) except Exception as e: - fail_count = fail_count + 1 source_dataset.error("FAILED: " + str(e)) if debug: diff --git a/zfs_autobackup/sync_snapshots.py b/zfs_autobackup/sync_snapshots.py index 366be2f..565bdc5 100644 --- a/zfs_autobackup/sync_snapshots.py +++ b/zfs_autobackup/sync_snapshots.py @@ -1,19 +1,21 @@ -from .ExecuteNode import ExecuteError +"""Snapshot-level replication: sync the snapshots of one dataset to a target dataset, +thinning out obsolete snapshots on both sides along the way.""" + from .ZfsBookmark import ZfsBookmark from .ZfsContainer import ZfsContainer from .ZfsPointInTime import ZfsPointInTime from .ZfsSnapshot import ZfsSnapshot -def _resolve_clone_origin(target_path, strip_path, guid_check, source_dataset, target_node): +def _resolve_clone_origin(source_dataset, target_node, target_path, strip_path, guid_check): """Return the source-side origin snapshot to use as zfs send -i base for a clone, or None if the clone relationship cannot be preserved on the target. + :type source_dataset: ZfsContainer + :type target_node: ZfsNode :type target_path: str :type strip_path: int :type guid_check: bool - :type source_dataset: ZfsContainer - :type target_node: ZfsNode :rtype: ZfsSnapshot|None """ @@ -26,35 +28,43 @@ def _resolve_clone_origin(target_path, strip_path, guid_check, source_dataset, t # to exist for the descendant, so replication can't preserve the relationship. origin_parent_path = source_origin_snap.name.split('@', 1)[0] if origin_parent_path == source_dataset.name or origin_parent_path.startswith(source_dataset.name + "/"): - source_dataset.warning("Cannot replicate as clone: origin '{}' lives on a namespace descendant of this dataset. Falling back to full send.".format(source_origin_snap.name)) + source_dataset.warning( + "Cannot replicate as clone: origin '{}' lives on a namespace descendant of this dataset. " + "Falling back to full send.".format(source_origin_snap.name)) return None try: target_origin_parent = source_origin_snap.parent.map_to_target_path(target_path, strip_path) except Exception as e: - source_dataset.warning("Cannot replicate as clone: cannot map origin '{}' to target ({}). Falling back to full send.".format(source_origin_snap.name, str(e))) + source_dataset.warning( + "Cannot replicate as clone: cannot map origin '{}' to target ({}). " + "Falling back to full send.".format(source_origin_snap.name, str(e))) return None target_origin_snap = target_node.get_snapshot(target_origin_parent + '@' + source_origin_snap.suffix) if not target_origin_snap.exists: - source_dataset.warning("Cannot replicate as clone: origin '{}' not available on target. Falling back to full send.".format(source_origin_snap.name)) + source_dataset.warning( + "Cannot replicate as clone: origin '{}' not available on target. " + "Falling back to full send.".format(source_origin_snap.name)) return None - if guid_check: - if not source_origin_snap.guid_matches(target_origin_snap): - source_dataset.warning("Cannot replicate as clone: origin guid mismatch between source and target. Falling back to full send.") - return None + if guid_check and not source_origin_snap.guid_matches(target_origin_snap): + source_dataset.warning( + "Cannot replicate as clone: origin guid mismatch between source and target. " + "Falling back to full send.") + return None return source_origin_snap -def _pre_clean(source_dataset, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): - """cleanup old stuff on the source before starting snapshot syncing +def _pre_clean(source_dataset, target_dataset, source_common_snapshot, source_obsoletes, target_obsoletes, + target_transfers): + """cleanup old stuff on source and target before starting snapshot syncing :type source_dataset: ZfsContainer - :type source_common_snapshot: ZfsSnapshot|ZfsBookmark|None :type target_dataset: ZfsContainer + :type source_common_snapshot: ZfsSnapshot|ZfsBookmark|None :type source_obsoletes: list[ZfsSnapshot] :type target_obsoletes: list[ZfsSnapshot] :type target_transfers: list[ZfsSnapshot] @@ -67,17 +77,20 @@ def _pre_clean(source_dataset, source_common_snapshot, target_dataset, source_ob and source_snapshot.find_snapshot_by_suffix(target_transfers) is None): source_snapshot.destroy() - # on target: destroy everything thats obsolete, except common_snapshot - if target_dataset.exists: - for target_snapshot in target_dataset.snapshots: - if (target_snapshot in target_obsoletes) \ - and (not source_common_snapshot or ( - target_snapshot.tagless_suffix != source_common_snapshot.tagless_suffix)): - if target_snapshot.exists: - target_snapshot.destroy() + # on target: destroy everything thats obsolete, except the common snapshot + if not target_dataset.exists: + return + + for target_snapshot in target_dataset.snapshots: + if (target_snapshot in target_obsoletes + and (not source_common_snapshot + or target_snapshot.tagless_suffix != source_common_snapshot.tagless_suffix) + and target_snapshot.exists): + target_snapshot.destroy() -def handle_incompatible_target(target_dataset, incompatible_target_snapshots, destroy_incompatible, force, source_common_snapshot, is_resume): +def _handle_incompatible_target(target_dataset, incompatible_target_snapshots, destroy_incompatible, force, + source_common_snapshot, is_resume): """destroy incompatible snapshots on target before sync, or inform user what to do :type target_dataset: ZfsContainer @@ -100,7 +113,7 @@ def handle_incompatible_target(target_dataset, incompatible_target_snapshots, de # its a dataset from an incremental send we probably can resume, so its fine return - #from this point on things get progressively worse.. + # from this point on things get progressively worse.. if source_common_snapshot: # we just need to delete some incompatibles: @@ -108,18 +121,16 @@ def handle_incompatible_target(target_dataset, incompatible_target_snapshots, de snapshot.warning("Incompatible snapshot") if not destroy_incompatible: - raise (Exception("Use --destroy-incompatible to get rid of these.")) - + raise Exception("Use --destroy-incompatible to get rid of these.") else: # no common snapshot, so the whole dataset is incompatible if incompatible_target_snapshots: if not force or not destroy_incompatible: target_dataset.error("Incompatible dataset!") - raise (Exception("Use --destroy-incompatible -F to overwrite the target dataset and start over.")) - else: - if not force: - target_dataset.error("Incompatible dataset!") - raise (Exception("Use -F to overwrite the target dataset and start over.")) + raise Exception("Use --destroy-incompatible -F to overwrite the target dataset and start over.") + elif not force: + target_dataset.error("Incompatible dataset!") + raise Exception("Use -F to overwrite the target dataset and start over.") target_dataset.warning("Overwriting incompatible dataset.") @@ -127,12 +138,11 @@ def handle_incompatible_target(target_dataset, incompatible_target_snapshots, de for snapshot in incompatible_target_snapshots: snapshot.destroy(fail_exception=True) - # target_dataset.invalidate_cache() target_dataset.rollback() def _validate_resume_token(source_dataset, target_dataset, start_snapshot): - """validate and get (or destroy) resume token + """validate and return the resume token, or abort (destroy) the resume state if its no longer valid :type source_dataset: ZfsContainer :type target_dataset: ZfsContainer @@ -140,24 +150,28 @@ def _validate_resume_token(source_dataset, target_dataset, start_snapshot): :rtype: str|None """ - if target_dataset.exists and 'receive_resume_token' in target_dataset.properties: - if start_snapshot is None: - target_dataset.verbose("Aborting resume, its obsolete.") - target_dataset.abort_resume() - else: - resume_token = target_dataset.properties['receive_resume_token'] - # not valid anymore? - resume_snapshot = source_dataset.zfs_node.get_resume_snapshot(resume_token) - if not resume_snapshot or start_snapshot.suffix != resume_snapshot.suffix: - target_dataset.verbose("Aborting resume, its no longer valid.") - target_dataset.abort_resume() - else: - return resume_token + if not target_dataset.exists or 'receive_resume_token' not in target_dataset.properties: + return None - return None + if start_snapshot is None: + target_dataset.verbose("Aborting resume, its obsolete.") + target_dataset.abort_resume() + return None + + resume_token = target_dataset.properties['receive_resume_token'] + + # not valid anymore? + resume_snapshot = source_dataset.zfs_node.get_resume_snapshot(resume_token) + if not resume_snapshot or start_snapshot.suffix != resume_snapshot.suffix: + target_dataset.verbose("Aborting resume, its no longer valid.") + target_dataset.abort_resume() + return None + + return resume_token -def _plan_sync(source_dataset, target_dataset, also_other_snapshots, guid_check, raw, bookmark_tag, required_snapshots=None): +def _plan_sync(source_dataset, target_dataset, also_other_snapshots, guid_check, raw, bookmark_tag, + required_snapshots=None): """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. :type source_dataset: ZfsContainer @@ -167,11 +181,11 @@ def _plan_sync(source_dataset, target_dataset, also_other_snapshots, guid_check, :type raw: bool :type bookmark_tag: str :type required_snapshots: set[ZfsSnapshot]|None - :rtype: ( ZfsSnapshot|ZfsBookmark, list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot] ) + :rtype: ( ZfsSnapshot|ZfsBookmark|None, list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot] ) Returns: tuple: A tuple containing: - - ZfsSnapshot|ZfsBookmark: The source common snapshot + - ZfsSnapshot|ZfsBookmark|None: The source common snapshot - list[ZfsSnapshot]: Our obsolete source snapshots, after transfer is done. (will be thinned asap) - list[ZfsSnapshot]: Our obsolete target snapshots, after transfer is done. (will be thinned asap) - list[ZfsSnapshot]: Transfer target snapshots. These need to be transferred. @@ -179,71 +193,129 @@ def _plan_sync(source_dataset, target_dataset, also_other_snapshots, guid_check, """ ### 1: determine common and start snapshot if target already exists: - if target_dataset.exists: target_dataset.debug("Determining start snapshot") - (source_common_snapshot, target_common_snapshot) = source_dataset.find_common_snapshot(target_dataset, - guid_check=guid_check, - bookmark_tag=bookmark_tag) + (source_common_snapshot, target_common_snapshot) = source_dataset.find_common_snapshot( + target_dataset, guid_check=guid_check, bookmark_tag=bookmark_tag) incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(target_common_snapshot, raw) else: source_common_snapshot = None incompatible_target_snapshots = [] # let thinner decide whats obsolete on source after the transfer is done - source_obsoletes = [] if source_dataset.our_snapshots: source_obsoletes = source_dataset.thin_list()[1] + else: + source_obsoletes = [] - ### 2: Determine possible target snapshots + ### 2: determine possible target snapshots # start with snapshots that already exist, minus incompatibles if target_dataset.exists: - possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots if - snapshot not in incompatible_target_snapshots] + possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots + if snapshot not in incompatible_target_snapshots] else: possible_target_snapshots = [] # add all snapshots from the source, starting after the common snapshot if it exists if source_common_snapshot is not None: source_snapshot = source_dataset.find_next_snapshot(source_common_snapshot) + elif source_dataset.snapshots: + source_snapshot = source_dataset.snapshots[0] else: - if source_dataset.snapshots: - source_snapshot = source_dataset.snapshots[0] - else: - source_snapshot = None + source_snapshot = None while source_snapshot: # we want it? is_required = required_snapshots is not None and source_snapshot in required_snapshots if is_required and not source_snapshot.is_ours and not also_other_snapshots: source_snapshot.verbose("Including as clone origin for a selected clone") - if (also_other_snapshots or source_snapshot.is_ours or is_required) and not source_snapshot.is_snapshot_excluded: + if (also_other_snapshots or source_snapshot.is_ours or is_required) \ + and not source_snapshot.is_snapshot_excluded: # create virtual target snapshot target_snapshot = target_dataset.zfs_node.get_snapshot( target_dataset.name + source_snapshot.typed_suffix, force_exists=False) possible_target_snapshots.append(target_snapshot) source_snapshot = source_dataset.find_next_snapshot(source_snapshot) - ### 3: Let the thinner decide what it wants by looking at all the possible target_snaphots at once. + ### 3: let the thinner decide what it wants by looking at all the possible target snapshots at once. # always keep the last target snapshot as common snapshot. if possible_target_snapshots: - (target_keeps, target_obsoletes) = target_dataset.zfs_node.thin_list(possible_target_snapshots, - keep_snapshots=[ - possible_target_snapshots[-1]]) + (target_keeps, target_obsoletes) = target_dataset.zfs_node.thin_list( + possible_target_snapshots, keep_snapshots=[possible_target_snapshots[-1]]) else: target_keeps = [] target_obsoletes = [] - ### 4: Look at what the thinner wants and create a list of snapshots we still need to transfer - target_transfers = [] - for target_keep in target_keeps: - if not target_keep.exists: - target_transfers.append(target_keep) + ### 4: look at what the thinner wants to keep and create a list of snapshots we still need to transfer + target_transfers = [target_keep for target_keep in target_keeps if not target_keep.exists] return source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots +def _active_properties(source_dataset, filter_properties, set_properties, property_format): + """determine the property filter/set lists to use for this transfer + + :type source_dataset: ZfsContainer + :type filter_properties: list[str] + :type set_properties: list[str] + :type property_format: str + :rtype: (list[str], list[str]) + """ + + (active_filter_properties, active_set_properties) = source_dataset.get_allowed_properties(filter_properties, + set_properties) + + # always filter properties that start with the property-format prefix (https://github.com/psy0rz/zfs_autobackup/issues/221) + property_prefix = property_format.split(':')[0] + for prop in source_dataset.properties: + if prop.startswith(property_prefix): + active_filter_properties.append(prop) + + return active_filter_properties, active_set_properties + + +def _post_transfer_source_cleanup(source_snapshot, prev_base, source_obsoletes, holds, use_bookmarks, bookmark_tag): + """after source_snapshot was transferred: hold or bookmark it as the new common snapshot, and + release/destroy the previous one if its obsolete. Returns the source-side point-in-time to use + as base for the next incremental send. + + :type source_snapshot: ZfsSnapshot + :type prev_base: ZfsPointInTime|None + :type source_obsoletes: list[ZfsSnapshot] + :type holds: bool + :type use_bookmarks: bool + :type bookmark_tag: str + :rtype: ZfsPointInTime + """ + + if use_bookmarks: + # bookmark the new common snapshot, so the snapshot itself can be destroyed when obsolete + source_bookmark = source_snapshot.bookmark(bookmark_tag) + if source_snapshot in source_obsoletes: + source_snapshot.destroy() + + # TODO: make a better is_ours specially for bookmarks, as part of the next refactoring splitting in more classes + # delete the previous bookmark if its ours (ends in our tag_separator + tag) + if isinstance(prev_base, ZfsBookmark) and prev_base.name.endswith( + source_snapshot.zfs_node.tag_seperator + bookmark_tag): + prev_base.destroy() + + return source_bookmark + + # not using bookmarks: hold the new common snapshot instead + if holds: + source_snapshot.hold() + + # release hold and cleanup the previous common snapshot if its obsolete + if isinstance(prev_base, ZfsSnapshot): + prev_base.release() + if prev_base in source_obsoletes: + prev_base.destroy() + + return source_snapshot + + def sync_snapshots(source_dataset, target_dataset, features, show_progress, filter_properties, set_properties, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, @@ -279,10 +351,9 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt :type required_snapshots: set[ZfsSnapshot]|None """ - # defaults for these settings if there is no encryption stuff going on: + # determine how to handle encryption during the transfer: send_properties = True raw = False - write_embedded = True # source dataset encrypted? if source_dataset.properties.get('encryption', 'off') != 'off': @@ -306,13 +377,11 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt # NOTE: we do a pre-clean because we dont want filesystems to fillup when backups keep failing. # Also usefull with no_send to still cleanup stuff. _pre_clean( - source_dataset, - source_common_snapshot=source_common_snapshot, target_dataset=target_dataset, - target_transfers=target_transfers, target_obsoletes=target_obsoletes, source_obsoletes=source_obsoletes) + source_dataset, target_dataset=target_dataset, source_common_snapshot=source_common_snapshot, + source_obsoletes=source_obsoletes, target_obsoletes=target_obsoletes, target_transfers=target_transfers) # check if we can resume if len(target_transfers) > 0: - if show_progress: source_dataset.zfs_node.logger.progress("Verifying resume token...") @@ -324,43 +393,43 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt source_dataset.zfs_node.logger.progress("Preparing...") # handle incompatible stuff on target - handle_incompatible_target(target_dataset, incompatible_target_snapshots, destroy_incompatible, force, source_common_snapshot, resume_token is not None) + _handle_incompatible_target(target_dataset, incompatible_target_snapshots, destroy_incompatible, force, + source_common_snapshot, resume_token is not None) # now actually transfer the snapshots, if we want if no_send or len(target_transfers) == 0: return - (active_filter_properties, active_set_properties) = source_dataset.get_allowed_properties(filter_properties, - set_properties) - - # always filter properties that start with the property-format prefix (https://github.com/psy0rz/zfs_autobackup/issues/221) - property_prefix = property_format.split(':')[0] - for prop in source_dataset.properties: - if prop.startswith(property_prefix): - active_filter_properties.append(prop) + (active_filter_properties, active_set_properties) = _active_properties(source_dataset, filter_properties, + set_properties, property_format) # encrypt at target? + write_embedded = True if encrypt and not raw: # filter out encryption properties to let encryption on the target take place active_filter_properties.extend(["keylocation", "pbkdf2iters", "keyformat", "encryption"]) write_embedded = False - # now actually transfer the snapshots - do_rollback = rollback # When the target dataset is being created fresh and the source is a clone whose # origin's target equivalent exists, send the first snapshot as an incremental from # that origin so zfs recv reconstructs the clone relationship on the target. # Only resolve this now (not earlier) — checking target origin existence is only # valid when we know this is actually a new full transfer. if source_common_snapshot is None and not no_clone: - clone_origin_snapshot = _resolve_clone_origin(target_path, strip_path, guid_check, source_dataset, target_dataset.zfs_node) + clone_origin_snapshot = _resolve_clone_origin(source_dataset, target_dataset.zfs_node, target_path, + strip_path, guid_check) else: clone_origin_snapshot = None + + # the source-side point-in-time (snapshot or bookmark) to use as base for the next incremental send if clone_origin_snapshot is not None: - prev_source_snapshot_bookmark = clone_origin_snapshot + incremental_base = clone_origin_snapshot else: - prev_source_snapshot_bookmark = source_common_snapshot + incremental_base = source_common_snapshot + prev_target_snapshot = target_dataset.find_snapshot(source_common_snapshot) + + do_rollback = rollback for target_snapshot in target_transfers: source_snapshot = source_dataset.find_snapshot(target_snapshot) @@ -369,13 +438,13 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt # do the rollback, one time at first transfer if do_rollback: if show_progress: - source_dataset.zfs_node.logger.progress(f"Rolling back {target_dataset}...") + source_dataset.zfs_node.logger.progress("Rolling back {}...".format(target_dataset)) target_dataset.rollback() do_rollback = False source_snapshot.transfer_snapshot(target_snapshot, features=features, - prev_snapshot=prev_source_snapshot_bookmark, show_progress=show_progress, + prev_snapshot=incremental_base, show_progress=show_progress, filter_properties=active_filter_properties, set_properties=active_set_properties, ignore_recv_exit_code=ignore_recv_exit_code, @@ -383,6 +452,7 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt send_properties=send_properties, send_pipes=send_pipes, recv_pipes=recv_pipes, zfs_compressed=zfs_compressed, force=force) + # the resume token is only valid for the first transfer resume_token = None # hold/release common snapshot on the target. @@ -392,40 +462,13 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt if prev_target_snapshot: prev_target_snapshot.release() - if use_bookmarks: - # bookmark common snapshot, and clean up obsolete snapshots and bookmark - source_bookmark = source_snapshot.bookmark(bookmark_tag) - if source_snapshot in source_obsoletes: - source_snapshot.destroy() - - # TODO: make a better is_ours specially for bookmarks, as part of the next refactoring splitting in more classes - # delete any bookmark that ends in ours tag_separator + tag. - if prev_source_snapshot_bookmark and type( - prev_source_snapshot_bookmark) is ZfsBookmark and prev_source_snapshot_bookmark.name.endswith( - source_dataset.zfs_node.tag_seperator + bookmark_tag): - prev_source_snapshot_bookmark.destroy() - - # dont use bookmarks - else: - source_bookmark = None - if holds: - source_snapshot.hold() - - # release hold, cleanup obsolete snapshot - if isinstance(prev_source_snapshot_bookmark, ZfsSnapshot): - prev_source_snapshot_bookmark.release() - if prev_source_snapshot_bookmark in source_obsoletes: - prev_source_snapshot_bookmark.destroy() + # cleanup on the source and determine the base for the next incremental send + incremental_base = _post_transfer_source_cleanup(source_snapshot, incremental_base, source_obsoletes, + holds, use_bookmarks, bookmark_tag) # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, # the rest was already destroyed or will not be send) if prev_target_snapshot is not None and prev_target_snapshot in target_obsoletes: prev_target_snapshot.destroy() - # we always try to use the bookmark during incremental send - if source_bookmark: - prev_source_snapshot_bookmark = source_bookmark - else: - prev_source_snapshot_bookmark = source_snapshot - prev_target_snapshot = target_snapshot From 072c2ee260204b44afb56b6fc53c00a6072f6f0b Mon Sep 17 00:00:00 2001 From: DatuX Date: Wed, 10 Jun 2026 10:17:46 +0200 Subject: [PATCH 128/135] refactor --- zfs_autobackup/sync_snapshots.py | 59 +++++++++++++++++++------------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/zfs_autobackup/sync_snapshots.py b/zfs_autobackup/sync_snapshots.py index 565bdc5..e02b562 100644 --- a/zfs_autobackup/sync_snapshots.py +++ b/zfs_autobackup/sync_snapshots.py @@ -90,7 +90,7 @@ def _pre_clean(source_dataset, target_dataset, source_common_snapshot, source_ob def _handle_incompatible_target(target_dataset, incompatible_target_snapshots, destroy_incompatible, force, - source_common_snapshot, is_resume): + source_common_snapshot, is_clone_origin, is_resume): """destroy incompatible snapshots on target before sync, or inform user what to do :type target_dataset: ZfsContainer @@ -98,9 +98,15 @@ def _handle_incompatible_target(target_dataset, incompatible_target_snapshots, d :type destroy_incompatible: bool :type force: bool :type source_common_snapshot: ZfsPointInTime|None + :type is_clone_origin: bool :type is_resume: bool """ + # a clone origin is not a real common snapshot of this dataset: it lives on a different + # target dataset, so it doesnt make existing content of this target dataset compatible. + if is_clone_origin: + source_common_snapshot = None + if not target_dataset.exists: # no target yet, so everything ok return @@ -171,7 +177,7 @@ def _validate_resume_token(source_dataset, target_dataset, start_snapshot): def _plan_sync(source_dataset, target_dataset, also_other_snapshots, guid_check, raw, bookmark_tag, - required_snapshots=None): + no_clone, target_path, strip_path, required_snapshots=None): """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. :type source_dataset: ZfsContainer @@ -180,12 +186,16 @@ def _plan_sync(source_dataset, target_dataset, also_other_snapshots, guid_check, :type guid_check: bool :type raw: bool :type bookmark_tag: str + :type no_clone: bool + :type target_path: str + :type strip_path: int :type required_snapshots: set[ZfsSnapshot]|None - :rtype: ( ZfsSnapshot|ZfsBookmark|None, list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot] ) + :rtype: ( ZfsSnapshot|ZfsBookmark|None, bool, list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot] ) Returns: tuple: A tuple containing: - - ZfsSnapshot|ZfsBookmark|None: The source common snapshot + - ZfsSnapshot|ZfsBookmark|None: The source common snapshot. (the clone origin snapshot when is_clone_origin is True) + - bool: is_clone_origin: the common snapshot is actually the clone origin, which lives on a different dataset. - list[ZfsSnapshot]: Our obsolete source snapshots, after transfer is done. (will be thinned asap) - list[ZfsSnapshot]: Our obsolete target snapshots, after transfer is done. (will be thinned asap) - list[ZfsSnapshot]: Transfer target snapshots. These need to be transferred. @@ -250,7 +260,17 @@ def _plan_sync(source_dataset, target_dataset, also_other_snapshots, guid_check, ### 4: look at what the thinner wants to keep and create a list of snapshots we still need to transfer target_transfers = [target_keep for target_keep in target_keeps if not target_keep.exists] - return source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots + ### 5: no common snapshot, but the source is a clone? then the origin acts as the common snapshot: + # the first send becomes an incremental from the origin, so zfs recv reconstructs the clone + # relationship on the target. + is_clone_origin = False + if source_common_snapshot is None and not no_clone and target_transfers: + source_common_snapshot = _resolve_clone_origin(source_dataset, target_dataset.zfs_node, target_path, + strip_path, guid_check) + is_clone_origin = source_common_snapshot is not None + + return (source_common_snapshot, is_clone_origin, source_obsoletes, target_obsoletes, target_transfers, + incompatible_target_snapshots) def _active_properties(source_dataset, filter_properties, set_properties, property_format): @@ -365,10 +385,11 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt # keep data encrypted by sending it raw (including properties) raw = True - (source_common_snapshot, source_obsoletes, target_obsoletes, target_transfers, + (source_common_snapshot, is_clone_origin, source_obsoletes, target_obsoletes, target_transfers, incompatible_target_snapshots) = \ _plan_sync(source_dataset, target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, guid_check=guid_check, raw=raw, bookmark_tag=bookmark_tag, + no_clone=no_clone, target_path=target_path, strip_path=strip_path, required_snapshots=required_snapshots) if show_progress: @@ -394,7 +415,7 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt # handle incompatible stuff on target _handle_incompatible_target(target_dataset, incompatible_target_snapshots, destroy_incompatible, force, - source_common_snapshot, resume_token is not None) + source_common_snapshot, is_clone_origin, resume_token is not None) # now actually transfer the snapshots, if we want if no_send or len(target_transfers) == 0: @@ -410,24 +431,16 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt active_filter_properties.extend(["keylocation", "pbkdf2iters", "keyformat", "encryption"]) write_embedded = False - # When the target dataset is being created fresh and the source is a clone whose - # origin's target equivalent exists, send the first snapshot as an incremental from - # that origin so zfs recv reconstructs the clone relationship on the target. - # Only resolve this now (not earlier) — checking target origin existence is only - # valid when we know this is actually a new full transfer. - if source_common_snapshot is None and not no_clone: - clone_origin_snapshot = _resolve_clone_origin(source_dataset, target_dataset.zfs_node, target_path, - strip_path, guid_check) - else: - clone_origin_snapshot = None + # the source-side point-in-time (snapshot, bookmark or clone origin) to use as base for the + # next incremental send + incremental_base = source_common_snapshot - # the source-side point-in-time (snapshot or bookmark) to use as base for the next incremental send - if clone_origin_snapshot is not None: - incremental_base = clone_origin_snapshot + # the target-side counterpart of the common snapshot. (a clone origin has no counterpart here: + # it lives on the origin's target dataset) + if is_clone_origin: + prev_target_snapshot = None else: - incremental_base = source_common_snapshot - - prev_target_snapshot = target_dataset.find_snapshot(source_common_snapshot) + prev_target_snapshot = target_dataset.find_snapshot(source_common_snapshot) do_rollback = rollback for target_snapshot in target_transfers: From 57a4071e3b391d335e8bc3c7c629c14e109399aa Mon Sep 17 00:00:00 2001 From: DatuX Date: Wed, 10 Jun 2026 10:56:40 +0200 Subject: [PATCH 129/135] remove print --- zfs_autobackup/ZfsContainer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py index b2cb108..2cde684 100644 --- a/zfs_autobackup/ZfsContainer.py +++ b/zfs_autobackup/ZfsContainer.py @@ -273,7 +273,6 @@ class ZfsContainer(ZfsDataset): def rollback(self): """rollback to latest existing snapshot on this dataset""" - print (f"HIJ IS {self._snapshots_bookmarks}") for snapshot in reversed(self.snapshots): if snapshot.exists: From 4e6e5e56ba261a0929c565de47774f676b8b0cdc Mon Sep 17 00:00:00 2001 From: DatuX Date: Wed, 10 Jun 2026 11:10:55 +0200 Subject: [PATCH 130/135] suppress progress reporting if needed --- zfs_autobackup/CliBase.py | 2 ++ zfs_autobackup/LogConsole.py | 7 +++++++ zfs_autobackup/sync_snapshots.py | 12 ++++-------- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/zfs_autobackup/CliBase.py b/zfs_autobackup/CliBase.py index 2bac095..e32d2fa 100644 --- a/zfs_autobackup/CliBase.py +++ b/zfs_autobackup/CliBase.py @@ -52,6 +52,8 @@ class CliBase(object): self.log = LogConsole(show_debug=args.debug, show_verbose=args.verbose, color=sys.stdout.isatty()) + self.log.show_progress=args.progress + self.verbose(self.HEADER) self.verbose("") diff --git a/zfs_autobackup/LogConsole.py b/zfs_autobackup/LogConsole.py index 51b66c9..acd905f 100644 --- a/zfs_autobackup/LogConsole.py +++ b/zfs_autobackup/LogConsole.py @@ -21,6 +21,8 @@ class LogConsole: self.last_log = "" self.show_debug = show_debug self.show_verbose = show_verbose + self.show_progress=True + self._progress_uncleared = False self.colorama = color and _COLORAMA_AVAILABLE @@ -64,6 +66,8 @@ class LogConsole: sys.stdout.flush() def progress(self, txt, current=None, total=None, failed=None): + if not self.show_progress: + return if current is not None: self._current=current @@ -91,6 +95,9 @@ class LogConsole: sys.stderr.flush() def clear_progress(self): + if not self.show_progress: + return + if self._progress_uncleared: if self.colorama: print(colorama.ansi.clear_line(), end='', file=sys.stderr) diff --git a/zfs_autobackup/sync_snapshots.py b/zfs_autobackup/sync_snapshots.py index e02b562..49b8eb2 100644 --- a/zfs_autobackup/sync_snapshots.py +++ b/zfs_autobackup/sync_snapshots.py @@ -392,8 +392,7 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt no_clone=no_clone, target_path=target_path, strip_path=strip_path, required_snapshots=required_snapshots) - if show_progress: - source_dataset.zfs_node.logger.progress("Pre-cleaning..") + source_dataset.zfs_node.logger.progress("Pre-cleaning..") # NOTE: we do a pre-clean because we dont want filesystems to fillup when backups keep failing. # Also usefull with no_send to still cleanup stuff. @@ -403,15 +402,13 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt # check if we can resume if len(target_transfers) > 0: - if show_progress: - source_dataset.zfs_node.logger.progress("Verifying resume token...") + source_dataset.zfs_node.logger.progress("Verifying resume token...") resume_token = _validate_resume_token(source_dataset, target_dataset, target_transfers[0]) else: resume_token = None - if show_progress: - source_dataset.zfs_node.logger.progress("Preparing...") + source_dataset.zfs_node.logger.progress("Preparing...") # handle incompatible stuff on target _handle_incompatible_target(target_dataset, incompatible_target_snapshots, destroy_incompatible, force, @@ -450,8 +447,7 @@ def sync_snapshots(source_dataset, target_dataset, features, show_progress, filt # do the rollback, one time at first transfer if do_rollback: - if show_progress: - source_dataset.zfs_node.logger.progress("Rolling back {}...".format(target_dataset)) + source_dataset.zfs_node.logger.progress("Rolling back {}...".format(target_dataset)) target_dataset.rollback() do_rollback = False From 8e51ff5e29c1df1e24160ddd8ad51a455265d6fd Mon Sep 17 00:00:00 2001 From: DatuX Date: Fri, 12 Jun 2026 21:47:59 +0200 Subject: [PATCH 131/135] beta4 --- zfs_autobackup/CliBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zfs_autobackup/CliBase.py b/zfs_autobackup/CliBase.py index e32d2fa..1c3f7f7 100644 --- a/zfs_autobackup/CliBase.py +++ b/zfs_autobackup/CliBase.py @@ -10,7 +10,7 @@ class CliBase(object): Overridden in subclasses that add stuff for the specific programs.""" # also used by setup.py - VERSION = "4.0-beta3" + VERSION = "4.0-beta4" HEADER = "{} v{} - (c)2026 E.H.Eefting (edwin@datux.nl)".format(os.path.basename(sys.argv[0]), VERSION) def __init__(self, argv, print_arguments=True): From d9f250501f7d07eaf3aaadbcd601070fa750afba Mon Sep 17 00:00:00 2001 From: DatuX Date: Sun, 21 Jun 2026 15:41:18 +0200 Subject: [PATCH 132/135] fix --- zfs_autobackup/sync_datasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zfs_autobackup/sync_datasets.py b/zfs_autobackup/sync_datasets.py index ce69601..390b4d9 100644 --- a/zfs_autobackup/sync_datasets.py +++ b/zfs_autobackup/sync_datasets.py @@ -256,7 +256,7 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta # NOTE: bookmark_written seems to be needed. (only 'bookmarks' was not enough on ubuntu 20) use_bookmarks = not no_bookmarks - if use_bookmarks and 'bookmark_written' not in common_features: + if use_bookmarks and 'bookmarks' not in common_features: source_dataset.warning("Disabling bookmarks, not supported on both pools.") use_bookmarks = False From da97719ad307e6bf074a7bdc0e954c9a9645d00a Mon Sep 17 00:00:00 2001 From: DatuX Date: Thu, 25 Jun 2026 13:16:55 +0200 Subject: [PATCH 133/135] only warn one time per pool that bookmarks are not supported --- zfs_autobackup/ZfsPool.py | 3 +++ zfs_autobackup/sync_datasets.py | 12 +++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/zfs_autobackup/ZfsPool.py b/zfs_autobackup/ZfsPool.py index 78bf5b8..0bc3485 100644 --- a/zfs_autobackup/ZfsPool.py +++ b/zfs_autobackup/ZfsPool.py @@ -30,6 +30,9 @@ class ZfsPool(): def verbose(self, txt): self.zfs_node.verbose("zpool {}: {}".format(self.name, txt)) + def warning(self, txt): + self.zfs_node.warning("zpool {}: {}".format(self.name, txt)) + def error(self, txt): self.zfs_node.error("zpool {}: {}".format(self.name, txt)) diff --git a/zfs_autobackup/sync_datasets.py b/zfs_autobackup/sync_datasets.py index 390b4d9..227f243 100644 --- a/zfs_autobackup/sync_datasets.py +++ b/zfs_autobackup/sync_datasets.py @@ -229,6 +229,7 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta fail_count = 0 target_datasets = [] + bookmarks_unsupported_pools = set() # source pools we've already warned about for count, source_dataset in enumerate(source_datasets, start=1): logger.progress("Analysing dataset...", count, len(source_datasets), fail_count) @@ -240,7 +241,6 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta target_datasets.append(target_dataset) # ensure parents exists - # TODO: this isnt perfect yet, in some cases it can create parents when it shouldn't. target_parent = target_dataset.parent if not no_send \ and target_parent is not None \ @@ -249,15 +249,17 @@ def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_ta target_dataset.debug("Creating unmountable parents") target_parent.create_filesystem(parents=True) - # determine common zpool features (cached, so no problem we call it often) - source_features = source_node.get_pool(source_dataset).features + # determine common zpool features + source_pool = source_node.get_pool(source_dataset) + source_features = source_pool.features target_features = target_node.get_pool(target_dataset).features common_features = [feature for feature in source_features if feature in target_features] - # NOTE: bookmark_written seems to be needed. (only 'bookmarks' was not enough on ubuntu 20) use_bookmarks = not no_bookmarks if use_bookmarks and 'bookmarks' not in common_features: - source_dataset.warning("Disabling bookmarks, not supported on both pools.") + if source_pool.name not in bookmarks_unsupported_pools: + source_pool.warning("Disabling bookmarks, not supported on both pools.") + bookmarks_unsupported_pools.add(source_pool.name) use_bookmarks = False # sync the snapshots of this dataset From c25fabed2eb903fbab5452aeb38023019df9cc11 Mon Sep 17 00:00:00 2001 From: DatuX Date: Wed, 26 Aug 2026 20:30:53 +0200 Subject: [PATCH 134/135] Update version from 4.0-beta4 to 4.0-rc1 --- zfs_autobackup/CliBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zfs_autobackup/CliBase.py b/zfs_autobackup/CliBase.py index 1c3f7f7..1d9b862 100644 --- a/zfs_autobackup/CliBase.py +++ b/zfs_autobackup/CliBase.py @@ -10,7 +10,7 @@ class CliBase(object): Overridden in subclasses that add stuff for the specific programs.""" # also used by setup.py - VERSION = "4.0-beta4" + VERSION = "4.0-rc1" HEADER = "{} v{} - (c)2026 E.H.Eefting (edwin@datux.nl)".format(os.path.basename(sys.argv[0]), VERSION) def __init__(self, argv, print_arguments=True): From f0faa85bc159276c91e42dc73341db75c8e16ce5 Mon Sep 17 00:00:00 2001 From: DatuX Date: Wed, 26 Aug 2026 20:54:30 +0200 Subject: [PATCH 135/135] Update README to remove v4.0 development notes Removed notes about the v4.0 development branch and its readiness for testing. --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index ae85a5d..0445109 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,5 @@ # ZFS autobackup -# NOTE: this is the v4.0 development branch. Not ready for testing yet! - -**This now will be the upcoming v4.0 release due to some major changes in how zfs-autobackup handles certain stuff** - [![Tests](https://github.com/psy0rz/zfs_autobackup/workflows/Regression%20tests/badge.svg)](https://github.com/psy0rz/zfs_autobackup/actions?query=workflow%3A%22Regression+tests%22) [![Coverage Status](https://coveralls.io/repos/github/psy0rz/zfs_autobackup/badge.svg)](https://coveralls.io/github/psy0rz/zfs_autobackup) [![Python Package](https://github.com/psy0rz/zfs_autobackup/workflows/Upload%20Python%20Package/badge.svg)](https://pypi.org/project/zfs-autobackup/) [![CodeQL](https://github.com/psy0rz/zfs_autobackup/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/psy0rz/zfs_autobackup/actions/workflows/codeql-analysis.yml)