diff --git a/tests/test_zfsnode.py b/tests/test_zfsnode.py index 723e978..4d64e00 100644 --- a/tests/test_zfsnode.py +++ b/tests/test_zfsnode.py @@ -15,7 +15,7 @@ class TestZfsNode(unittest2.TestCase): node = ZfsNode("test", logger, description=description) with self.subTest("first snapshot"): - node.consistent_snapshot(node.selected_datasets(exclude_paths=[], exclude_received=False), "test-1", 100000) + node.consistent_snapshot(node.selected_datasets(exclude_paths=[], exclude_received=False, exclude_unchanged=False, min_change=200000), "test-1", 100000) r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) self.assertEqual(r, """ test_source1 @@ -33,7 +33,7 @@ test_target1 """) with self.subTest("second snapshot, no changes, no snapshot"): - node.consistent_snapshot(node.selected_datasets(exclude_paths=[], exclude_received=False), "test-2", 1) + node.consistent_snapshot(node.selected_datasets(exclude_paths=[], exclude_received=False, exclude_unchanged=False, min_change=200000), "test-2", 1) r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) self.assertEqual(r, """ test_source1 @@ -51,7 +51,7 @@ test_target1 """) with self.subTest("second snapshot, no changes, empty snapshot"): - node.consistent_snapshot(node.selected_datasets(exclude_paths=[], exclude_received=False), "test-2", 0) + node.consistent_snapshot(node.selected_datasets(exclude_paths=[], exclude_received=False, exclude_unchanged=False, min_change=200000), "test-2", 0) r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) self.assertEqual(r, """ test_source1 @@ -79,7 +79,7 @@ test_target1 with self.subTest("Test if all cmds are executed correctly (no failures)"): with OutputIO() as buf: with redirect_stdout(buf): - node.consistent_snapshot(node.selected_datasets(exclude_paths=[], exclude_received=False), "test-1", + node.consistent_snapshot(node.selected_datasets(exclude_paths=[], exclude_received=False, exclude_unchanged=False, min_change=1), "test-1", 0, pre_snapshot_cmds=["echo pre1", "echo pre2"], post_snapshot_cmds=["echo post1 >&2", "echo post2 >&2"] @@ -95,7 +95,7 @@ test_target1 with OutputIO() as buf: with redirect_stdout(buf): with self.assertRaises(ExecuteError): - node.consistent_snapshot(node.selected_datasets(exclude_paths=[], exclude_received=False), "test-1", + node.consistent_snapshot(node.selected_datasets(exclude_paths=[], exclude_received=False, exclude_unchanged=False, min_change=1), "test-1", 0, pre_snapshot_cmds=["echo pre1", "false", "echo pre2"], post_snapshot_cmds=["echo post1", "false", "echo post2"] @@ -112,7 +112,7 @@ test_target1 with redirect_stdout(buf): with self.assertRaises(ExecuteError): #same snapshot name as before so it fails - node.consistent_snapshot(node.selected_datasets(exclude_paths=[], exclude_received=False), "test-1", + node.consistent_snapshot(node.selected_datasets(exclude_paths=[], exclude_received=False, exclude_unchanged=False, min_change=1), "test-1", 0, pre_snapshot_cmds=["echo pre1", "echo pre2"], post_snapshot_cmds=["echo post1", "echo post2"] @@ -126,10 +126,19 @@ test_target1 def test_getselected(self): + + # should be excluded by property + shelltest("zfs create test_source1/fs1/subexcluded") + shelltest("zfs set autobackup:test=false test_source1/fs1/subexcluded") + + # should be excluded by being unchanged + shelltest("zfs create test_source1/fs1/unchanged") + shelltest("zfs snapshot test_source1/fs1/unchanged@somesnapshot") + logger = LogStub() description = "[Source]" node = ZfsNode("test", logger, description=description) - s = pformat(node.selected_datasets(exclude_paths=[], exclude_received=False)) + s = pformat(node.selected_datasets(exclude_paths=[], exclude_received=False, exclude_unchanged=True, min_change=1)) print(s) # basics @@ -137,11 +146,6 @@ test_target1 (local): test_source1/fs1/sub, (local): test_source2/fs2/sub]""") - # caching, so expect same result after changing it - subprocess.check_call("zfs set autobackup:test=true test_source2/fs3", shell=True) - self.assertEqual(s, """[(local): test_source1/fs1, - (local): test_source1/fs1/sub, - (local): test_source2/fs2/sub]""") def test_validcommand(self): logger = LogStub() diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 5b47d2a..f29c5e5 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -11,12 +11,10 @@ from zfs_autobackup.ZfsNode import ZfsNode from zfs_autobackup.ThinnerRule import ThinnerRule - - class ZfsAutobackup: """main class""" - VERSION = "3.1" + VERSION = "3.1.1-beta1" HEADER = "zfs-autobackup v{} - (c)2021 E.H.Eefting (edwin@datux.nl)".format(VERSION) def __init__(self, argv, print_arguments=True): @@ -63,18 +61,16 @@ class ZfsAutobackup: 'default)s)') parser.add_argument('--allow-empty', action='store_true', help='If nothing has changed, still create empty snapshots. (same as --min-change=0)') - parser.add_argument('--ignore-replicated', action='store_true', - help='Ignore datasets that seem to be replicated some other way. (No changes since ' - 'lastest snapshot. Useful for proxmox HA replication)') + + parser.add_argument('--ignore-replicated', action='store_true', help=argparse.SUPPRESS) + parser.add_argument('--exclude-unchanged', action='store_true', + help='Exclude datasets that have no changes since any last snapshot. (Useful in combination with proxmox HA replication)') parser.add_argument('--exclude-received', action='store_true', - help='Ignore datasets that have the origin of their autobackup: property as "received". ' - 'This can avoid recursive replication between two backup partners. You would usually ' - 'use --ignore-replicated instead of this option.') + help='Exclude datasets that have the origin of their autobackup: property as "received". ' + 'This can avoid recursive replication between two backup partners.') parser.add_argument('--strip-path', metavar='N', default=0, type=int, help='Number of directories to strip from target path (use 1 when cloning zones between 2 ' 'SmartOS machines)') - # parser.add_argument('--buffer', default="", help='Use mbuffer with specified size to speedup zfs transfer. - # (e.g. --buffer 1G) Will also show nice progress output.') parser.add_argument('--clear-refreservation', action='store_true', help='Filter "refreservation" property. (recommended, safes space. same as ' @@ -125,16 +121,19 @@ class ZfsAutobackup: parser.add_argument('--resume', action='store_true', help=argparse.SUPPRESS) parser.add_argument('--raw', action='store_true', help=argparse.SUPPRESS) - #these things all do stuff by piping zfs send/recv IO + # these things all do stuff by piping zfs send/recv IO parser.add_argument('--send-pipe', metavar="COMMAND", default=[], action='append', help='pipe zfs send output through COMMAND (can be used multiple times)') parser.add_argument('--recv-pipe', metavar="COMMAND", default=[], action='append', help='pipe zfs recv input through COMMAND (can be used multiple times)') - parser.add_argument('--compress', metavar='TYPE', default=None, nargs='?', const='zstd-adapt', choices=compressors.choices(), help='Use compression during transfer, defaults to zstd-adapt if TYPE is not specified. ({})'.format(", ".join(compressors.choices()))) - parser.add_argument('--rate', metavar='DATARATE', default=None, help='Limit data transfer rate (e.g. 128K. requires mbuffer.)') - parser.add_argument('--buffer', metavar='SIZE', default=None, help='Add zfs send and recv buffers to smooth out IO bursts. (e.g. 128M. requires mbuffer)') - - + parser.add_argument('--compress', metavar='TYPE', default=None, nargs='?', const='zstd-adapt', + choices=compressors.choices(), + help='Use compression during transfer, defaults to zstd-adapt if TYPE is not specified. ({})'.format( + ", ".join(compressors.choices()))) + parser.add_argument('--rate', metavar='DATARATE', default=None, + help='Limit data transfer rate (e.g. 128K. requires mbuffer.)') + parser.add_argument('--buffer', metavar='SIZE', default=None, + help='Add zfs send and recv buffers to smooth out IO bursts. (e.g. 128M. requires mbuffer)') # note args is the only global variable we use, since its a global readonly setting anyway args = parser.parse_args(argv) @@ -177,6 +176,10 @@ class ZfsAutobackup: if args.compress and args.zfs_compressed: self.warning("Using --compress with --zfs-compressed, might be inefficient.") + if args.ignore_replicated: + self.warning("--ignore-replicated has been renamed, using --exclude-unchanged") + args.exclude_unchanged = True + def verbose(self, txt): self.log.verbose(txt) @@ -289,12 +292,12 @@ class ZfsAutobackup: def get_send_pipes(self, logger): """determine the zfs send pipe""" - ret=[] + ret = [] # IO buffer if self.args.buffer: logger("zfs send buffer : {}".format(self.args.buffer)) - ret.extend([ ExecuteNode.PIPE, "mbuffer", "-q", "-s128k", "-m"+self.args.buffer ]) + ret.extend([ExecuteNode.PIPE, "mbuffer", "-q", "-s128k", "-m" + self.args.buffer]) # custom pipes for send_pipe in self.args.send_pipe: @@ -303,27 +306,26 @@ class ZfsAutobackup: logger("zfs send custom pipe : {}".format(send_pipe)) # compression - if self.args.compress!=None: + if self.args.compress != None: ret.append(ExecuteNode.PIPE) - cmd=compressors.compress_cmd(self.args.compress) + cmd = compressors.compress_cmd(self.args.compress) ret.extend(cmd) logger("zfs send compression : {}".format(" ".join(cmd))) # transfer rate if self.args.rate: logger("zfs send transfer rate : {}".format(self.args.rate)) - ret.extend([ ExecuteNode.PIPE, "mbuffer", "-q", "-s128k", "-m16M", "-R"+self.args.rate ]) - + ret.extend([ExecuteNode.PIPE, "mbuffer", "-q", "-s128k", "-m16M", "-R" + self.args.rate]) return ret def get_recv_pipes(self, logger): - ret=[] + ret = [] # decompression - if self.args.compress!=None: - cmd=compressors.decompress_cmd(self.args.compress) + if self.args.compress != None: + cmd = compressors.decompress_cmd(self.args.compress) ret.extend(cmd) ret.append(ExecuteNode.PIPE) logger("zfs recv decompression : {}".format(" ".join(cmd))) @@ -336,10 +338,10 @@ class ZfsAutobackup: # IO buffer if self.args.buffer: - #only add second buffer if its usefull. (e.g. non local transfer or other pipes active) - if self.args.ssh_source!=None or self.args.ssh_target!=None or self.args.recv_pipe or self.args.send_pipe or self.args.compress!=None: + # only add second buffer if its usefull. (e.g. non local transfer or other pipes active) + if self.args.ssh_source != None or self.args.ssh_target != None or self.args.recv_pipe or self.args.send_pipe or self.args.compress != None: logger("zfs recv buffer : {}".format(self.args.buffer)) - ret.extend(["mbuffer", "-q", "-s128k", "-m"+self.args.buffer, ExecuteNode.PIPE ]) + ret.extend(["mbuffer", "-q", "-s128k", "-m" + self.args.buffer, ExecuteNode.PIPE]) return ret @@ -351,8 +353,8 @@ class ZfsAutobackup: :type source_node: ZfsNode """ - send_pipes=self.get_send_pipes(source_node.verbose) - recv_pipes=self.get_recv_pipes(target_node.verbose) + send_pipes = self.get_send_pipes(source_node.verbose) + recv_pipes = self.get_recv_pipes(target_node.verbose) fail_count = 0 count = 0 @@ -392,7 +394,8 @@ class ZfsAutobackup: no_send=self.args.no_send, destroy_incompatible=self.args.destroy_incompatible, send_pipes=send_pipes, recv_pipes=recv_pipes, - decrypt=self.args.decrypt, encrypt=self.args.encrypt, zfs_compressed=self.args.zfs_compressed ) + decrypt=self.args.decrypt, encrypt=self.args.encrypt, + zfs_compressed=self.args.zfs_compressed) except Exception as e: fail_count = fail_count + 1 source_dataset.error("FAILED: " + str(e)) @@ -418,20 +421,6 @@ class ZfsAutobackup: for source_dataset in source_datasets: source_dataset.thin(skip_holds=True) - def filter_replicated(self, datasets): - if not self.args.ignore_replicated: - return datasets - else: - self.set_title("Filtering already replicated filesystems") - ret = [] - for dataset in datasets: - if dataset.is_changed(self.args.min_change): - ret.append(dataset) - else: - dataset.verbose("Ignoring, already replicated") - - return ret - def filter_properties_list(self): if self.args.filter_properties: @@ -487,7 +476,7 @@ class ZfsAutobackup: # may still need to be used to explicitly exclude a backup with the 'received' source to avoid accidental # recursive replication of a zvol that is currently being received in another session (as it will have changes). exclude_paths = [] - exclude_received=self.args.exclude_received + exclude_received = self.args.exclude_received if self.args.ssh_source == self.args.ssh_target: if self.args.target_path: # target and source are the same, make sure to exclude target_path @@ -495,21 +484,19 @@ class ZfsAutobackup: exclude_paths.append(self.args.target_path) else: self.warning("Source and target are on the same host, excluding received datasets from selection.") - exclude_received=True + exclude_received = True - - selected_source_datasets = source_node.selected_datasets(exclude_received=exclude_received, - exclude_paths=exclude_paths) - if not selected_source_datasets: + source_datasets = source_node.selected_datasets(exclude_received=exclude_received, + exclude_paths=exclude_paths, + exclude_unchanged=self.args.exclude_unchanged, + min_change=self.args.min_change) + if not source_datasets: self.error( "No source filesystems selected, please do a 'zfs set autobackup:{0}=true' on the source datasets " "you want to select.".format( self.args.backup_name)) return 255 - # filter out already replicated stuff? - source_datasets = self.filter_replicated(selected_source_datasets) - ################# snapshotting if not self.args.no_snapshot: self.set_title("Snapshotting") diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 089df2b..83e8e91 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -112,7 +112,7 @@ class ZfsDataset: """true if this dataset is a snapshot""" return self.name.find("@") != -1 - def is_selected(self, value, source, inherited, exclude_received, exclude_paths): + def is_selected(self, value, source, inherited, exclude_received, exclude_paths, exclude_unchanged, min_change): """determine if dataset should be selected for backup (called from ZfsNode) @@ -122,6 +122,12 @@ class ZfsDataset: :type source: str :type inherited: bool :type exclude_received: bool + :type exclude_unchanged: bool + :type min_change: bool + + :param value: Value of the zfs property ("false"/"true"/"child"/"-") + :param source: Source of the zfs property ("local"/"received", "-") + :param inherited: True of the value/source was inherited from a higher dataset. """ # sanity checks @@ -135,28 +141,40 @@ class ZfsDataset: raise (Exception( "{} autobackup-property has illegal value: '{}'".format(self.name, value))) + # non specified, ignore + if value == "-": + return False + + # only select childs of this dataset, ignore + if value == "child" and not inherited: + return False + + # manually excluded by property + if value == "false": + self.verbose("Excluded") + return False + + # from here on the dataset is selected by property, now do additional exclusion checks + # our path starts with one of the excluded paths? for exclude_path in exclude_paths: if self.name.startswith(exclude_path): # too noisy for verbose - self.debug("Excluded (in exclude list)") + self.debug("Excluded (path in exclude list)") return False - # now determine if its actually selected - if value == "false": - self.verbose("Excluded (disabled)") + if source == "received": + if exclude_received: + self.verbose("Excluded (dataset already received)") + return False + + if exclude_unchanged and not self.is_changed(min_change): + self.verbose("Excluded (unchanged since last snapshot)") return False - elif value == "true" or (value == "child" and inherited): - if source == "local": - self.verbose("Selected") - return True - elif source == "received": - if exclude_received: - self.verbose("Excluded (dataset already received)") - return False - else: - self.verbose("Selected") - return True + + self.verbose("Selected") + return True + @CachedProperty def parent(self): @@ -296,6 +314,7 @@ class ZfsDataset: if min_changed_bytes == 0: return True + if int(self.properties['written']) < min_changed_bytes: return False else: diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index 50aacd7..937f20a 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -214,9 +214,7 @@ class ZfsNode(ExecuteNode): except Exception as e: pass - - - def selected_datasets(self, exclude_received, exclude_paths): + def selected_datasets(self, exclude_received, exclude_paths, exclude_unchanged, min_change): """determine filesystems that should be backed up by looking at the special autobackup-property, systemwide returns: list of ZfsDataset @@ -251,7 +249,7 @@ class ZfsNode(ExecuteNode): source = raw_source # determine it - if dataset.is_selected(value=value, source=source, inherited=inherited, exclude_received=exclude_received, exclude_paths=exclude_paths): + if dataset.is_selected(value=value, source=source, inherited=inherited, exclude_received=exclude_received, exclude_paths=exclude_paths, exclude_unchanged=exclude_unchanged, min_change=min_change): selected_filesystems.append(dataset) return selected_filesystems