From cab2f98bb8d022b87d92a8b726f40dd6ef213700 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 23 Feb 2022 17:47:50 +0100 Subject: [PATCH 1/2] Better strip path handling and collision checking. Now also supports stripping so much it ends up on a pool-target. Fixes #102, #117 --- tests/test_zfsautobackup.py | 7 +++++++ zfs_autobackup/ZfsAutobackup.py | 33 ++++++++++++++++++++++++++++++--- zfs_autobackup/ZfsDataset.py | 14 ++++++++++++-- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index e95c02b..858b7b2 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -418,6 +418,13 @@ test_target1/fs2/sub test_target1/fs2/sub@test-20101111000000 """) + def test_strippath_collision(self): + with self.assertRaisesRegexp(Exception,"collision"): + ZfsAutobackup("test test_target1 --verbose --strip-path=2 --no-progress --debug".split(" ")).run() + + def test_strippath_toomuch(self): + with self.assertRaisesRegexp(Exception,"too much"): + ZfsAutobackup("test test_target1 --verbose --strip-path=3 --no-progress --debug".split(" ")).run() def test_clearrefres(self): diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 0264a2e..061f42a 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -9,12 +9,12 @@ from .ZfsDataset import ZfsDataset from .LogConsole import LogConsole from .ZfsNode import ZfsNode from .ThinnerRule import ThinnerRule - +import os.path class ZfsAutobackup: """main class""" - VERSION = "3.1.1" + VERSION = "3.1.2-rc1" HEADER = "zfs-autobackup v{} - (c)2021 E.H.Eefting (edwin@datux.nl)".format(VERSION) def __init__(self, argv, print_arguments=True): @@ -364,6 +364,29 @@ class ZfsAutobackup: return ret + def make_target_name(self, source_dataset): + """make target_name from a source_dataset""" + stripped=source_dataset.lstrip_path(self.args.strip_path) + if stripped!="": + return self.args.target_path + "/" + stripped + else: + return self.args.target_path + + def check_target_names(self, source_node, source_datasets, target_node): + """check all target names for collesions etc due to strip-options""" + + self.debug("Checking target names:") + target_datasets={} + for source_dataset in source_datasets: + + target_name = self.make_target_name(source_dataset) + source_dataset.debug("-> {}".format(target_name)) + + if target_name in target_datasets: + raise Exception("Target collision: Target path {} encountered twice, due to: {} and {}".format(target_name, source_dataset, target_datasets[target_name])) + + target_datasets[target_name]=source_dataset + # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: def sync_datasets(self, source_node, source_datasets, target_node): """Sync datasets, or thin-only on both sides @@ -387,13 +410,14 @@ class ZfsAutobackup: try: # determine corresponding target_dataset - target_name = self.args.target_path + "/" + source_dataset.lstrip_path(self.args.strip_path) + target_name = self.make_target_name(source_dataset) target_dataset = ZfsDataset(target_node, target_name) target_datasets.append(target_dataset) # ensure parents exists # TODO: this isnt perfect yet, in some cases it can create parents when it shouldn't. if not self.args.no_send \ + and target_dataset.parent \ and target_dataset.parent not in target_datasets \ and not target_dataset.parent.exists: target_dataset.parent.create_filesystem(parents=True) @@ -560,6 +584,9 @@ class ZfsAutobackup: raise (Exception( "Target path '{}' does not exist. Please create this dataset first.".format(target_dataset))) + # check for collisions due to strip-path + self.check_target_names(source_node, source_datasets, target_node) + # do the actual sync # NOTE: even with no_send, no_thinning and no_snapshot it does a usefull thing because it checks if the common snapshots and shows incompatible snapshots fail_count = self.sync_datasets( diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index da56b54..394fcfa 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -79,7 +79,11 @@ class ZfsDataset: Args: :type count: int """ - return "/".join(self.split_path()[count:]) + components=self.split_path() + if count>len(components): + raise Exception("Trying to strip too much from path ({} items from {})".format(count, self.name)) + + return "/".join(components[count:]) def rstrip_path(self, count): """return name with last count components stripped @@ -188,7 +192,11 @@ class ZfsDataset: if self.is_snapshot: return ZfsDataset(self.zfs_node, self.filesystem_name) else: - return ZfsDataset(self.zfs_node, self.rstrip_path(1)) + stripped=self.rstrip_path(1) + if stripped: + return ZfsDataset(self.zfs_node, stripped) + else: + return None # NOTE: unused for now # def find_prev_snapshot(self, snapshot, also_other_snapshots=False): @@ -1007,6 +1015,8 @@ class ZfsDataset: :type destroy_incompatible: bool """ + self.verbose("sending to {}".format(target_dataset)) + (common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_keeps, incompatible_target_snapshots) = \ self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots) From e4356cb5167e4daca917ad13d4f8095f82cfd704 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 23 Feb 2022 18:36:03 +0100 Subject: [PATCH 2/2] Added -F (--force) to allow 1:1 replication. --- zfs_autobackup/ZfsAutobackup.py | 6 ++++-- zfs_autobackup/ZfsDataset.py | 15 ++++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 061f42a..3eea16d 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -14,7 +14,7 @@ import os.path class ZfsAutobackup: """main class""" - VERSION = "3.1.2-rc1" + VERSION = "3.1.2-rc2" HEADER = "zfs-autobackup v{} - (c)2021 E.H.Eefting (edwin@datux.nl)".format(VERSION) def __init__(self, argv, print_arguments=True): @@ -87,6 +87,8 @@ class ZfsAutobackup: parser.add_argument('--rollback', action='store_true', help='Rollback changes to the latest target snapshot before starting. (normally you can ' 'prevent changes by setting the readonly property on the target_path to on)') + parser.add_argument('--force', '-F', action='store_true', + help='Use zfs -F option to force overwrite/rollback. (Usefull with --strip-path=1, but use with care)') parser.add_argument('--destroy-incompatible', action='store_true', help='Destroy incompatible snapshots on target. Use with care! (implies --rollback)') parser.add_argument('--destroy-missing', metavar="SCHEDULE", type=str, default=None, @@ -438,7 +440,7 @@ class ZfsAutobackup: destroy_incompatible=self.args.destroy_incompatible, send_pipes=send_pipes, recv_pipes=recv_pipes, decrypt=self.args.decrypt, encrypt=self.args.encrypt, - zfs_compressed=self.args.zfs_compressed) + zfs_compressed=self.args.zfs_compressed, force=self.args.force) except Exception as e: fail_count = fail_count + 1 source_dataset.error("FAILED: " + str(e)) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 394fcfa..a08fa9f 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -585,7 +585,7 @@ class ZfsDataset: return output_pipe - def recv_pipe(self, pipe, features, recv_pipes, filter_properties=None, set_properties=None, ignore_exit_code=False): + def recv_pipe(self, pipe, features, recv_pipes, filter_properties=None, set_properties=None, ignore_exit_code=False, force=False): """starts a zfs recv for this snapshot and uses pipe as input note: you can it both on a snapshot or filesystem object. The @@ -626,6 +626,9 @@ class ZfsDataset: # verbose output cmd.append("-v") + if force: + cmd.append("-F") + if 'extensible_dataset' in features and "-s" in self.zfs_node.supported_recv_options: # support resuming self.debug("Enabled resume support") @@ -656,7 +659,7 @@ class ZfsDataset: def transfer_snapshot(self, target_snapshot, features, prev_snapshot, show_progress, filter_properties, set_properties, ignore_recv_exit_code, resume_token, - raw, send_properties, write_embedded, send_pipes, recv_pipes, zfs_compressed): + raw, send_properties, write_embedded, send_pipes, recv_pipes, zfs_compressed, force): """transfer this snapshot to target_snapshot. specify prev_snapshot for incremental transfer @@ -697,7 +700,7 @@ class ZfsDataset: pipe = self.send_pipe(features=features, show_progress=show_progress, prev_snapshot=prev_snapshot, resume_token=resume_token, raw=raw, send_properties=send_properties, write_embedded=write_embedded, send_pipes=send_pipes, zfs_compressed=zfs_compressed) target_snapshot.recv_pipe(pipe, features=features, filter_properties=filter_properties, - set_properties=set_properties, ignore_exit_code=ignore_recv_exit_code, recv_pipes=recv_pipes) + set_properties=set_properties, ignore_exit_code=ignore_recv_exit_code, recv_pipes=recv_pipes, force=force) def abort_resume(self): """abort current resume state""" @@ -994,7 +997,7 @@ class ZfsDataset: def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, - no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed): + no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force): """sync this dataset's snapshots to target_dataset, while also thinning out old snapshots along the way. @@ -1079,7 +1082,9 @@ class ZfsDataset: filter_properties=active_filter_properties, set_properties=active_set_properties, ignore_recv_exit_code=ignore_recv_exit_code, - resume_token=resume_token, write_embedded=write_embedded, raw=raw, send_properties=send_properties, send_pipes=send_pipes, recv_pipes=recv_pipes, zfs_compressed=zfs_compressed) + resume_token=resume_token, write_embedded=write_embedded, raw=raw, + send_properties=send_properties, send_pipes=send_pipes, + recv_pipes=recv_pipes, zfs_compressed=zfs_compressed, force=force) resume_token = None