zfs-autoverify wip (basics start to function)

tests/test_verify.py

@@ -32,6 +32,8 @@ class TestZfsEncryption(unittest2.TestCase):
         shelltest("zfs create -V 1M test_source1/fs1/ok_zvol")
         shelltest("dd if=/dev/urandom of=/dev/zvol/test_source1/fs1/ok_zvol count=1 bs=512k")
 
+        shelltest("zfs create -V 1M test_source1/fs1/bad_zvol")
+        shelltest("dd if=/dev/urandom of=/dev/zvol/test_source1/fs1/bad_zvol count=1 bs=512k")
 
         #create backup
         with patch('time.strftime', return_value="test-20101111000000"):
@@ -44,10 +46,15 @@ class TestZfsEncryption(unittest2.TestCase):
         shelltest("echo >> /test_target1/test_source1/fs1/bad_filesystem/test_verify.py")
         shelltest("zfs snapshot test_target1/test_source1/fs1/bad_filesystem@test-20101111000000")
 
+        #do the same hack for the bad zvol
+        shelltest("zfs destroy test_target1/test_source1/fs1/bad_zvol@test-20101111000000")
+        shelltest("dd if=/dev/urandom of=/dev/zvol/test_target1/test_source1/fs1/bad_zvol count=1 bs=1")
+        shelltest("zfs snapshot test_target1/test_source1/fs1/bad_zvol@test-20101111000000")
+
+
         # make sure we cant accidently compare current data
         shelltest("zfs mount test_target1/test_source1/fs1/ok_filesystem")
         shelltest("rm /test_source1/fs1/ok_filesystem/*")
-        # shelltest("zfs mount /test_target1/test_source1/fs1/bad_filesystem")
         shelltest("rm /test_source1/fs1/bad_filesystem/*")
         shelltest("dd if=/dev/zero of=/dev/zvol/test_source1/fs1/ok_zvol count=1 bs=512k")
 
@@ -56,7 +63,7 @@ class TestZfsEncryption(unittest2.TestCase):
         self.assertFalse(ZfsAutoverify("test test_target1 --verbose --test".split(" ")).run())
 
         #rsync mode
-        self.assertEqual(1, ZfsAutoverify("test test_target1 --verbose".split(" ")).run())
+        self.assertEqual(2, ZfsAutoverify("test test_target1 --verbose".split(" ")).run())
-        self.assertEqual(1, ZfsAutoverify("test test_target1 --ssh-source=localhost --verbose --exclude-received".split(" ")).run())
+        self.assertEqual(2, ZfsAutoverify("test test_target1 --ssh-source=localhost --verbose --exclude-received".split(" ")).run())
-        self.assertEqual(1, ZfsAutoverify("test test_target1 --ssh-target=localhost --verbose --exclude-received".split(" ")).run())
+        self.assertEqual(2, ZfsAutoverify("test test_target1 --ssh-target=localhost --verbose --exclude-received".split(" ")).run())
 

zfs_autobackup/ZfsAutoverify.py

@@ -37,12 +37,17 @@ class ZfsAutoverify(ZfsAuto):
 
         return parser
 
+    def compare_trees_tar(self , source_node, source_path, target_node, target_path):
+        """compare two trees using tar. compatible and simple"""
+
+        self.error("XXX implement")
+        pass
+
+
     def compare_trees_rsync(self , source_node, source_path, target_node, target_path):
-        """recursively compare checksums in both directory trees"""
+        """use rsync to compare two trees.
-
+         Advantage is that we can see which individual files differ.
-        #currently we use rsync for this.
+         But requires rsync and cant do remote to remote."""
-        #NOTE: perhaps support multiple compare implementations?
-
 
         cmd = ["rsync", "-rcn", "--info=COPY,DEL,MISC,NAME,SYMSAFE", "--msgs2stderr", "--delete" ]
 
@@ -78,8 +83,7 @@ class ZfsAutoverify(ZfsAuto):
 
         try:
 
-
+            # mount the snapshots
-            #mount the snapshots
             source_snapshot.mount(source_mnt)
             target_snapshot.mount(target_mnt)
 
@@ -89,9 +93,42 @@ class ZfsAutoverify(ZfsAuto):
             source_snapshot.unmount()
             target_snapshot.unmount()
 
-    def verify_volume(self, source_dataset, target_dataset):
+    def hash_dev(self, node, dev):
-        target_dataset.error("XXX implement me")
+        """calculate md5sum of a device on a node"""
-        pass
+
+        node.debug("Hashing {} ".format(dev))
+
+        cmd = [ "md5sum", dev ]
+
+        stdout = node.run(cmd)
+
+        if node.readonly:
+            hashed=None
+        else:
+            hashed = stdout[0].split(" ")[0]
+
+        node.debug("Hash of {} is {}".format(dev, hashed))
+
+        return hashed
+
+    def verify_volume(self, source_dataset, source_snapshot, target_dataset, target_snapshot):
+        """compare the contents of two zfs volume snapshots"""
+
+        try:
+            #make sure the volume snapshot is visible in /dev
+            source_dataset.set("snapdev", "visible")
+            target_dataset.set("snapdev", "visible")
+
+            source_hash=self.hash_dev(source_snapshot.zfs_node, "/dev/zvol/"+source_snapshot.name)
+            target_hash=self.hash_dev(target_snapshot.zfs_node, "/dev/zvol/"+target_snapshot.name)
+
+            if source_hash!=target_hash:
+                raise Exception("md5hash difference: {} != {}".format(source_hash, target_hash))
+
+        finally:
+            source_dataset.inherit("snapdev")
+            target_dataset.inherit("snapdev")
+
 
     def verify_datasets(self, source_mnt, source_datasets, target_node, target_mnt):
 
@@ -121,7 +158,7 @@ class ZfsAutoverify(ZfsAuto):
                 if source_dataset.properties['type']=="filesystem":
                     self.verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt)
                 elif source_dataset.properties['type']=="volume":
-                    self.verify_volume(source_dataset, target_dataset)
+                    self.verify_volume(source_dataset, source_snapshot, target_dataset, target_snapshot)
                 else:
                     raise(Exception("{} has unknown type {}".format(source_dataset, source_dataset.properties['type'])))
 

zfs_autobackup/ZfsDataset.py

@@ -1119,3 +1119,44 @@ class ZfsDataset:
         ]
 
         self.zfs_node.run(cmd=cmd, valid_exitcodes=[0])
+
+    # unused/untested for now
+    # def clone(self, name):
+    #     """clones this snapshot and returns ZfsDataset of the clone"""
+    #
+    #     self.debug("Cloning to {}".format(name))
+    #
+    #     cmd = [
+    #         "zfs", "clone", self.name, name
+    #     ]
+    #
+    #     self.zfs_node.run(cmd=cmd, valid_exitcodes=[0])
+    #
+    #     return ZfsDataset(self.zfs_node, name, force_exists=True)
+
+    def set(self, prop, value):
+        """set a zfs property"""
+
+        self.debug("Setting {}={}".format(prop, value))
+
+        cmd = [
+            "zfs", "set", "{}={}".format(prop, value), self.name
+        ]
+
+        self.zfs_node.run(cmd=cmd, valid_exitcodes=[0])
+
+        self.invalidate()
+
+    def inherit(self, prop):
+        """inherit zfs property"""
+
+        self.debug("Inheriting property {}".format(prop))
+
+        cmd = [
+            "zfs", "inherit", prop, self.name
+        ]
+
+        self.zfs_node.run(cmd=cmd, valid_exitcodes=[0])
+
+        self.invalidate()
+