zfs-autoverify wip

tests/test_verify.py

@@ -5,7 +5,8 @@ from basetest import *
 # test zfs-verify:
 # - when there is no common snapshot at all
 # - when encryption key not loaded
-# - test mode
+# - --test mode
+# - --fs-compare methods
 # - on snapshots of datasets:
 #   - that are correct
 #   - that are different
@@ -21,21 +22,41 @@ class TestZfsEncryption(unittest2.TestCase):
     def setUp(self):
         prepare_zpools()
 
+        #create actual test files and data
         shelltest("zfs create test_source1/fs1/ok_filesystem")
-        shelltest("cp *.py /test_source1/fs1/ok_filesystem")
+        shelltest("cp tests/*.py /test_source1/fs1/ok_filesystem")
+
+        shelltest("zfs create test_source1/fs1/bad_filesystem")
+        shelltest("cp tests/*.py /test_source1/fs1/bad_filesystem")
+
         shelltest("zfs create -V 1M test_source1/fs1/ok_zvol")
         shelltest("dd if=/dev/urandom of=/dev/zvol/test_source1/fs1/ok_zvol count=1 bs=512k")
 
+
+        #create backup
         with patch('time.strftime', return_value="test-20101111000000"):
-            self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress".split(" ")).run())
+            self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --no-holds".split(" ")).run())
+
+        #Do an ugly hack to create a fault in the bad filesystem
+        #In zfs-autoverify it doenst matter that the snapshot isnt actually the same snapshot, so this hack works
+        shelltest("zfs destroy test_target1/test_source1/fs1/bad_filesystem@test-20101111000000")
+        shelltest("zfs mount test_target1/test_source1/fs1/bad_filesystem")
+        shelltest("echo >> /test_target1/test_source1/fs1/bad_filesystem/test_verify.py")
+        shelltest("zfs snapshot test_target1/test_source1/fs1/bad_filesystem@test-20101111000000")
 
         # make sure we cant accidently compare current data
+        shelltest("zfs mount test_target1/test_source1/fs1/ok_filesystem")
         shelltest("rm /test_source1/fs1/ok_filesystem/*")
+        # shelltest("zfs mount /test_target1/test_source1/fs1/bad_filesystem")
+        shelltest("rm /test_source1/fs1/bad_filesystem/*")
         shelltest("dd if=/dev/zero of=/dev/zvol/test_source1/fs1/ok_zvol count=1 bs=512k")
 
     def test_verify(self):
 
         self.assertFalse(ZfsAutoverify("test test_target1 --verbose --test".split(" ")).run())
 
-        self.assertFalse(ZfsAutoverify("test test_target1 --verbose".split(" ")).run())
+        #rsync mode
+        self.assertEqual(1, ZfsAutoverify("test test_target1 --verbose".split(" ")).run())
+        self.assertEqual(1, ZfsAutoverify("test test_target1 --ssh-source=localhost --verbose --exclude-received".split(" ")).run())
+        self.assertEqual(1, ZfsAutoverify("test test_target1 --ssh-target=localhost --verbose --exclude-received".split(" ")).run())
 

zfs_autobackup/ZfsAutoverify.py

@@ -30,14 +30,19 @@ class ZfsAutoverify(ZfsAuto):
         """extend common parser with  extra stuff needed for zfs-autobackup"""
 
         parser=super(ZfsAutoverify, self).get_parser()
+
+        group=parser.add_argument_group("Verify options")
+        group.add_argument('--fs-compare', metavar='METHOD', default="tar", choices=["tar", "rsync"],
+                            help='Compare method to use for filesystems. (tar, rsync) Default: %(default)s ')
+
         return parser
 
-    def compare_trees(self , source_node, source_path, target_node, target_path):
+    def compare_trees_rsync(self , source_node, source_path, target_node, target_path):
-        """recursively compare checksums in both trees"""
+        """recursively compare checksums in both directory trees"""
-
-        #NOTE: perhaps support multiple compare methods/commands?
 
         #currently we use rsync for this.
+        #NOTE: perhaps support multiple compare implementations?
+
 
         cmd = ["rsync", "-rcn", "--info=COPY,DEL,MISC,NAME,SYMSAFE", "--msgs2stderr", "--delete" ]
 
@@ -45,44 +50,50 @@ class ZfsAutoverify(ZfsAuto):
         if source_node.ssh_to is None and target_node.ssh_to is None:
             cmd.append("{}/".format(source_path))
             cmd.append("{}/".format(target_path))
+            source_node.debug("Running rsync locally, on source.")
             stdout, stderr = source_node.run(cmd, return_stderr=True)
 
         #source is local
         elif source_node.ssh_to is None and target_node.ssh_to is not None:
             cmd.append("{}/".format(source_path))
             cmd.append("{}:{}/".format(target_node.ssh_to, target_path))
+            source_node.debug("Running rsync locally, on source.")
             stdout, stderr = source_node.run(cmd, return_stderr=True)
 
         #target is local
         elif source_node.ssh_to is not None and target_node.ssh_to is None:
             cmd.append("{}:{}/".format(source_node.ssh_to, source_path))
             cmd.append("{}/".format(target_path))
-
+            source_node.debug("Running rsync locally, on target.")
             stdout, stderr=target_node.run(cmd, return_stderr=True)
 
         else:
-            raise Exception("Source and target cant both be remote when using rsync to verify datasets")
+            raise Exception("Source and target cant both be remote when verifying. (rsync limitation)")
 
         if stderr:
             raise Exception("Dataset verify failed, see above list for differences")
 
     def verify_filesystem(self, source_snapshot, source_mnt, target_snapshot, target_mnt):
+        """Compare the contents of two zfs filesystem snapshots """
 
         try:
+
+
+            #mount the snapshots
             source_snapshot.mount(source_mnt)
             target_snapshot.mount(target_mnt)
 
-            self.compare_trees(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt)
+            self.compare_trees_rsync(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt)
-
 
         finally:
             source_snapshot.unmount()
             target_snapshot.unmount()
 
     def verify_volume(self, source_dataset, target_dataset):
+        target_dataset.error("XXX implement me")
         pass
 
-    def verify_datasets(self, source_node, source_mnt, source_datasets, target_node, target_mnt):
+    def verify_datasets(self, source_mnt, source_datasets, target_node, target_mnt):
 
         fail_count=0
         count = 0
@@ -102,6 +113,9 @@ class ZfsAutoverify(ZfsAuto):
                 source_snapshot = source_dataset.find_common_snapshot(target_dataset)
                 target_snapshot = target_dataset.find_snapshot(source_snapshot)
 
+                if source_snapshot is None or target_snapshot is None:
+                    raise(Exception("Cant find common snapshot"))
+
                 target_snapshot.verbose("Verifying...")
 
                 if source_dataset.properties['type']=="filesystem":
@@ -183,7 +197,6 @@ class ZfsAutoverify(ZfsAuto):
             source_mnt, target_mnt=self.create_mountpoints(source_node, target_node)
 
             fail_count = self.verify_datasets(
-                source_node=source_node,
                 source_mnt=source_mnt,
                 source_datasets=source_datasets,
                 target_mnt=target_mnt,

zfs_autobackup/ZfsDataset.py

@@ -1102,6 +1102,8 @@ class ZfsDataset:
 
     def mount(self, mount_point):
 
+        self.debug("Mounting")
+
         cmd = [
             "mount", "-tzfs", self.name, mount_point
         ]
@@ -1110,6 +1112,8 @@ class ZfsDataset:
 
     def unmount(self):
 
+        self.debug("Unmounting")
+
         cmd = [
             "umount", self.name
         ]