diff --git a/tests/test_verify.py b/tests/test_verify.py index 30acd7e..7d2fb5e 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -67,7 +67,7 @@ class TestZfsEncryption(unittest2.TestCase): self.assertFalse(ZfsAutoverify("test test_target1 --verbose --test".split(" ")).run()) with self.subTest("rsync, remote source and target. (not supported, all 6 fail)"): - self.assertEqual(6, ZfsAutoverify("test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --exclude-received".split(" ")).run()) + self.assertEqual(6, ZfsAutoverify("test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --exclude-received --fs-compare=rsync".split(" ")).run()) def runchecked(testname, command): with self.subTest(testname): @@ -81,10 +81,17 @@ class TestZfsEncryption(unittest2.TestCase): self.assertRegex(buf.getvalue(), "bad_filesystem: FAILED:") self.assertRegex(buf.getvalue(), "bad_zvol: FAILED:") - runchecked("rsync, remote source", "test test_target1 --ssh-source=localhost --verbose --exclude-received") - runchecked("rsync, remote target", "test test_target1 --ssh-target=localhost --verbose --exclude-received") - runchecked("rsync, local", "test test_target1 --verbose --exclude-received") + runchecked("rsync, remote source", "test test_target1 --ssh-source=localhost --verbose --exclude-received --fs-compare=rsync") + runchecked("rsync, remote target", "test test_target1 --ssh-target=localhost --verbose --exclude-received --fs-compare=rsync") + runchecked("rsync, local", "test test_target1 --verbose --exclude-received --fs-compare=rsync") + runchecked("tar, remote source and remote target", + "test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --exclude-received --fs-compare=tar") + runchecked("tar, remote source", + "test test_target1 --ssh-source=localhost --verbose --exclude-received --fs-compare=tar") + runchecked("tar, remote target", + "test test_target1 --ssh-target=localhost --verbose --exclude-received --fs-compare=tar") + runchecked("tar, local", "test test_target1 --verbose --exclude-received --fs-compare=tar") with self.subTest("no common snapshot"): #destroy common snapshot, now 3 should fail diff --git a/zfs_autobackup/ExecuteNode.py b/zfs_autobackup/ExecuteNode.py index c5743a3..715067d 100644 --- a/zfs_autobackup/ExecuteNode.py +++ b/zfs_autobackup/ExecuteNode.py @@ -78,10 +78,11 @@ class ExecuteNode(LogStub): return self.ssh_to is None def run(self, cmd, inp=None, tab_split=False, valid_exitcodes=None, readonly=False, hide_errors=False, - return_stderr=False, pipe=False): + return_stderr=False, pipe=False, return_all=False): """run a command on the node , checks output and parses/handle output and returns it - Either uses a local shell (sh -c) or remote shell (ssh) to execute the command. Therefore the command can have stuff like actual pipes in it, if you dont want to use pipe=True to pipe stuff. + Either uses a local shell (sh -c) or remote shell (ssh) to execute the command. + Therefore the command can have stuff like actual pipes in it, if you dont want to use pipe=True to pipe stuff. :param cmd: the actual command, should be a list, where the first item is the command and the rest are parameters. use ExecuteNode.PIPE to add an unescaped | @@ -94,6 +95,7 @@ class ExecuteNode(LogStub): :param readonly: make this True if the command doesn't make any changes and is safe to execute in testmode :param hide_errors: don't show stderr output as error, instead show it as debugging output (use to hide expected errors) :param return_stderr: return both stdout and stderr as a tuple. (normally only returns stdout) + :param return_all: return both stdout and stderr and exit_code as a tuple. (normally only returns stdout) """ @@ -106,6 +108,7 @@ class ExecuteNode(LogStub): # stderr parser error_lines = [] + returned_exit_code=None def stderr_handler(line): if tab_split: @@ -155,7 +158,9 @@ class ExecuteNode(LogStub): if not cmd_pipe.execute(stdout_handler=stdout_handler): raise(ExecuteError("Last command returned error")) - if return_stderr: + if return_all: + return output_lines, error_lines, cmd_item.process and cmd_item.process.returncode + elif return_stderr: return output_lines, error_lines else: return output_lines diff --git a/zfs_autobackup/ZfsAutoverify.py b/zfs_autobackup/ZfsAutoverify.py index 883c301..550011a 100644 --- a/zfs_autobackup/ZfsAutoverify.py +++ b/zfs_autobackup/ZfsAutoverify.py @@ -1,11 +1,185 @@ import os +import time +from .ExecuteNode import ExecuteNode from .ZfsAuto import ZfsAuto from .ZfsDataset import ZfsDataset from .ZfsNode import ZfsNode import sys import platform +def hash_tree_tar(node, path): + """calculate md5sum of a directory tree, using tar""" + + node.debug("Hashing filesystem {} ".format(path)) + + cmd=[ "tar", "-cf", "-", "-C", path, ".", + ExecuteNode.PIPE, "md5sum"] + + stdout = node.run(cmd) + + if node.readonly: + hashed=None + else: + hashed = stdout[0].split(" ")[0] + + node.debug("Hash of {} filesytem is {}".format(path, hashed)) + + return hashed + + +def compare_trees_tar(source_node, source_path, target_node, target_path): + """compare two trees using tar. compatible and simple""" + + source_hash= hash_tree_tar(source_node, source_path) + target_hash= hash_tree_tar(target_node, target_path) + + if source_hash != target_hash: + raise Exception("md5hash difference: {} != {}".format(source_hash, target_hash)) + + +def compare_trees_rsync(source_node, source_path, target_node, target_path): + """use rsync to compare two trees. + Advantage is that we can see which individual files differ. + But requires rsync and cant do remote to remote.""" + + cmd = ["rsync", "-rcn", "--info=COPY,DEL,MISC,NAME,SYMSAFE", "--msgs2stderr", "--delete" ] + + #local + if source_node.ssh_to is None and target_node.ssh_to is None: + cmd.append("{}/".format(source_path)) + cmd.append("{}/".format(target_path)) + source_node.debug("Running rsync locally, on source.") + stdout, stderr = source_node.run(cmd, return_stderr=True) + + #source is local + elif source_node.ssh_to is None and target_node.ssh_to is not None: + cmd.append("{}/".format(source_path)) + cmd.append("{}:{}/".format(target_node.ssh_to, target_path)) + source_node.debug("Running rsync locally, on source.") + stdout, stderr = source_node.run(cmd, return_stderr=True) + + #target is local + elif source_node.ssh_to is not None and target_node.ssh_to is None: + cmd.append("{}:{}/".format(source_node.ssh_to, source_path)) + cmd.append("{}/".format(target_path)) + source_node.debug("Running rsync locally, on target.") + stdout, stderr=target_node.run(cmd, return_stderr=True) + + else: + raise Exception("Source and target cant both be remote when verifying. (rsync limitation)") + + if stderr: + raise Exception("Dataset verify failed, see above list for differences") + + +def verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt, method): + """Compare the contents of two zfs filesystem snapshots """ + + try: + + # mount the snapshots + source_snapshot.mount(source_mnt) + target_snapshot.mount(target_mnt) + + if method=='rsync': + compare_trees_rsync(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt) + elif method == 'tar': + compare_trees_tar(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt) + else: + raise(Exception("program errror, unknown method")) + + finally: + source_snapshot.unmount() + target_snapshot.unmount() + + +def hash_dev(node, dev): + """calculate md5sum of a device on a node""" + + node.debug("Hashing volume {} ".format(dev)) + + cmd = [ "md5sum", dev ] + + stdout = node.run(cmd) + + if node.readonly: + hashed=None + else: + hashed = stdout[0].split(" ")[0] + + node.debug("Hash of volume {} is {}".format(dev, hashed)) + + return hashed + +def activate_volume_snapshot(dataset, snapshot): + """enables snapdev, waits and tries to findout /dev path to the volume, in a compatible way. (linux/freebsd/smartos)""" + + dataset.set("snapdev", "visible") + + #NOTE: add smartos location to this list as well + locations=[ + "/dev/zvol/" + snapshot.name + ] + + dataset.debug("Waiting for /dev entry to appear...") + time.sleep(0.1) + + start_time=time.time() + while time.time()-start_time<10: + for location in locations: + stdout, stderr, exit_code=dataset.zfs_node.run(["test", "-e", location], return_all=True, valid_exitcodes=[0,1]) + + #fake it in testmode + if dataset.zfs_node.readonly: + return location + + if exit_code==0: + return location + time.sleep(1) + + raise(Exception("Timeout while waiting for {} entry to appear.".format(locations))) + +def deacitvate_volume_snapshot(dataset): + dataset.inherit("snapdev") + + +def verify_volume(source_dataset, source_snapshot, target_dataset, target_snapshot): + """compare the contents of two zfs volume snapshots""" + + try: + source_dev= activate_volume_snapshot(source_dataset, source_snapshot) + target_dev= activate_volume_snapshot(target_dataset, target_snapshot) + + source_hash= hash_dev(source_snapshot.zfs_node, source_dev) + target_hash= hash_dev(target_snapshot.zfs_node, target_dev) + + if source_hash!=target_hash: + raise Exception("md5hash difference: {} != {}".format(source_hash, target_hash)) + + finally: + deacitvate_volume_snapshot(source_dataset) + deacitvate_volume_snapshot(target_dataset) + + +def create_mountpoints(source_node, target_node): + + # prepare mount points + source_node.debug("Create temporary mount point") + source_mnt = "/tmp/zfs-autoverify_source_{}_{}".format(platform.node(), os.getpid()) + source_node.run(["mkdir", source_mnt]) + + target_node.debug("Create temporary mount point") + target_mnt = "/tmp/zfs-autoverify_target_{}_{}".format(platform.node(), os.getpid()) + target_node.run(["mkdir", target_mnt]) + + return source_mnt, target_mnt + + +def cleanup_mountpoint(node, mnt): + node.debug("Cleaning up temporary mount point") + node.run([ "rmdir", mnt ], hide_errors=True, valid_exitcodes=[] ) + class ZfsAutoverify(ZfsAuto): """The zfs-autoverify class, default agruments and stuff come from ZfsAuto""" @@ -37,105 +211,6 @@ class ZfsAutoverify(ZfsAuto): return parser - def compare_trees_tar(self , source_node, source_path, target_node, target_path): - """compare two trees using tar. compatible and simple""" - - self.error("XXX implement") - pass - - - def compare_trees_rsync(self , source_node, source_path, target_node, target_path): - """use rsync to compare two trees. - Advantage is that we can see which individual files differ. - But requires rsync and cant do remote to remote.""" - - cmd = ["rsync", "-rcn", "--info=COPY,DEL,MISC,NAME,SYMSAFE", "--msgs2stderr", "--delete" ] - - #local - if source_node.ssh_to is None and target_node.ssh_to is None: - cmd.append("{}/".format(source_path)) - cmd.append("{}/".format(target_path)) - source_node.debug("Running rsync locally, on source.") - stdout, stderr = source_node.run(cmd, return_stderr=True) - - #source is local - elif source_node.ssh_to is None and target_node.ssh_to is not None: - cmd.append("{}/".format(source_path)) - cmd.append("{}:{}/".format(target_node.ssh_to, target_path)) - source_node.debug("Running rsync locally, on source.") - stdout, stderr = source_node.run(cmd, return_stderr=True) - - #target is local - elif source_node.ssh_to is not None and target_node.ssh_to is None: - cmd.append("{}:{}/".format(source_node.ssh_to, source_path)) - cmd.append("{}/".format(target_path)) - source_node.debug("Running rsync locally, on target.") - stdout, stderr=target_node.run(cmd, return_stderr=True) - - else: - raise Exception("Source and target cant both be remote when verifying. (rsync limitation)") - - if stderr: - raise Exception("Dataset verify failed, see above list for differences") - - def verify_filesystem(self, source_snapshot, source_mnt, target_snapshot, target_mnt): - """Compare the contents of two zfs filesystem snapshots """ - - try: - - # mount the snapshots - source_snapshot.mount(source_mnt) - target_snapshot.mount(target_mnt) - - self.compare_trees_rsync(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt) - - finally: - source_snapshot.unmount() - target_snapshot.unmount() - - def hash_dev(self, node, dev): - """calculate md5sum of a device on a node""" - - node.debug("Hashing {} ".format(dev)) - - cmd = [ "md5sum", dev ] - - stdout = node.run(cmd) - - if node.readonly: - hashed=None - else: - hashed = stdout[0].split(" ")[0] - - node.debug("Hash of {} is {}".format(dev, hashed)) - - return hashed - - def verify_volume(self, source_dataset, source_snapshot, target_dataset, target_snapshot): - """compare the contents of two zfs volume snapshots""" - - try: - #make sure the volume snapshot is visible in /dev - source_dataset.set("snapdev", "visible") - target_dataset.set("snapdev", "visible") - - # fixme: not compatible with freebsd and others. - source_dev="/dev/zvol/"+source_snapshot.name - target_dev="/dev/zvol/"+target_snapshot.name - source_dataset.zfs_node.run(["udevadm", "trigger", source_dev]) - target_dataset.zfs_node.run(["udevadm", "trigger", target_dev]) - - source_hash=self.hash_dev(source_snapshot.zfs_node, source_dev) - target_hash=self.hash_dev(target_snapshot.zfs_node, target_dev) - - if source_hash!=target_hash: - raise Exception("md5hash difference: {} != {}".format(source_hash, target_hash)) - - finally: - source_dataset.inherit("snapdev") - target_dataset.inherit("snapdev") - - def verify_datasets(self, source_mnt, source_datasets, target_node, target_mnt): fail_count=0 @@ -162,9 +237,9 @@ class ZfsAutoverify(ZfsAuto): target_snapshot.verbose("Verifying...") if source_dataset.properties['type']=="filesystem": - self.verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt) + verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt, self.args.fs_compare) elif source_dataset.properties['type']=="volume": - self.verify_volume(source_dataset, source_snapshot, target_dataset, target_snapshot) + verify_volume(source_dataset, source_snapshot, target_dataset, target_snapshot) else: raise(Exception("{} has unknown type {}".format(source_dataset, source_dataset.properties['type']))) @@ -178,24 +253,6 @@ class ZfsAutoverify(ZfsAuto): return fail_count - def create_mountpoints(self, source_node, target_node): - - # prepare mount points - source_node.debug("Create temporary mount point") - source_mnt = "/tmp/zfs-autoverify_source_{}_{}".format(platform.node(), os.getpid()) - source_node.run(["mkdir", source_mnt]) - - target_node.debug("Create temporary mount point") - target_mnt = "/tmp/zfs-autoverify_target_{}_{}".format(platform.node(), os.getpid()) - target_node.run(["mkdir", target_mnt]) - - return source_mnt, target_mnt - - def cleanup_mountpoint(self, node, mnt): - node.debug("Cleaning up temporary mount point") - node.run([ "rmdir", mnt ], hide_errors=True, valid_exitcodes=[] ) - - def run(self): source_node=None @@ -237,7 +294,7 @@ class ZfsAutoverify(ZfsAuto): self.set_title("Verifying") - source_mnt, target_mnt=self.create_mountpoints(source_node, target_node) + source_mnt, target_mnt= create_mountpoints(source_node, target_node) fail_count = self.verify_datasets( source_mnt=source_mnt, @@ -273,10 +330,10 @@ class ZfsAutoverify(ZfsAuto): # cleanup if source_mnt is not None: - self.cleanup_mountpoint(source_node, source_mnt) + cleanup_mountpoint(source_node, source_mnt) if target_mnt is not None: - self.cleanup_mountpoint(target_node, target_mnt) + cleanup_mountpoint(target_node, target_mnt)