more tests

tests/test_blockhasher.py

@@ -0,0 +1,100 @@
+from basetest import *
+from zfs_autobackup.BlockHasher import BlockHasher
+
+# make VERY sure this works correctly under all circumstances.
+
+# sha1 sums of files, (bs=4096)
+# da39a3ee5e6b4b0d3255bfef95601890afd80709  empty
+# 642027d63bb0afd7e0ba197f2c66ad03e3d70de1  partial
+# 3c0bf91170d873b8e327d3bafb6bc074580d11b7  whole
+# 2e863f1fcccd6642e4e28453eba10d2d3f74d798  whole2
+# 959e6b58078f0cfd2fb3d37e978fda51820473ff  whole_whole2
+# 309ffffba2e1977d12f3b7469971f30d28b94bd8  whole_whole2_partial
+
+class TestBlockHahser(unittest2.TestCase):
+
+    def setUp(self):
+        pass
+
+    def test_empty(self):
+        block_hasher=BlockHasher(count=1)
+        self.assertEqual(
+            list(block_hasher.generate("tests/data/empty")),
+            []
+        )
+
+    def test_partial(self):
+        block_hasher = BlockHasher(count=1)
+        self.assertEqual(
+            list(block_hasher.generate("tests/data/partial")),
+            [(0, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1")]
+        )
+
+    def test_whole(self):
+        block_hasher = BlockHasher(count=1)
+        self.assertEqual(
+            list(block_hasher.generate("tests/data/whole")),
+            [(0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7")]
+        )
+
+    def test_whole2(self):
+        block_hasher = BlockHasher(count=1)
+        self.assertEqual(
+            list(block_hasher.generate("tests/data/whole_whole2")),
+            [
+                (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"),
+                (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798")
+            ]
+        )
+
+    def test_wwp(self):
+        block_hasher = BlockHasher(count=1)
+        self.assertEqual(
+            list(block_hasher.generate("tests/data/whole_whole2_partial")),
+            [
+                (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), #whole
+                (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), #whole2
+                (2, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1")  #partial
+            ]
+        )
+
+    def test_wwp_count2(self):
+        block_hasher = BlockHasher(count=2)
+        self.assertEqual(
+            list(block_hasher.generate("tests/data/whole_whole2_partial")),
+            [
+                (0, "959e6b58078f0cfd2fb3d37e978fda51820473ff"), #whole_whole2
+                (1, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1")  #partial
+            ]
+        )
+
+    def test_big(self):
+        block_hasher=BlockHasher(count=10)
+        self.assertEqual(
+            list(block_hasher.generate("tests/data/whole_whole2_partial")),
+            [
+                (0, "309ffffba2e1977d12f3b7469971f30d28b94bd8"), #whole_whole2_partial
+            ])
+
+    def test_blockhash_compare(self):
+
+
+        block_hasher=BlockHasher(count=1)
+        generator=block_hasher.generate("tests/data/whole_whole2_partial")
+        self.assertEqual(3,block_hasher.compare("tests/data/whole_whole2_partial", generator))
+
+        block_hasher=BlockHasher(count=1)
+        with self.assertRaisesRegexp(Exception, "^Block 1 mismatched!"):
+            generator=block_hasher.generate("tests/data/whole_whole2_partial")
+            self.assertEqual(3,block_hasher.compare("tests/data/whole", generator))
+
+        block_hasher=BlockHasher(count=10)
+        generator=block_hasher.generate("tests/data/whole_whole2_partial")
+        self.assertEqual(1,block_hasher.compare("tests/data/whole_whole2_partial", generator))
+
+        #different order to make sure seek functions
+        block_hasher=BlockHasher(count=1)
+        checksums=list(block_hasher.generate("tests/data/whole_whole2_partial"))
+        checksums.reverse()
+        self.assertEqual(3,block_hasher.compare("tests/data/whole_whole2_partial", checksums))
+

tests/test_treehasher.py

@@ -0,0 +1,58 @@
+from basetest import *
+from zfs_autobackup.BlockHasher import BlockHasher
+
+# sha1 sums of files, (bs=4096)
+# da39a3ee5e6b4b0d3255bfef95601890afd80709  empty
+# 642027d63bb0afd7e0ba197f2c66ad03e3d70de1  partial
+# 3c0bf91170d873b8e327d3bafb6bc074580d11b7  whole
+# 2e863f1fcccd6642e4e28453eba10d2d3f74d798  whole2
+# 959e6b58078f0cfd2fb3d37e978fda51820473ff  whole_whole2
+# 309ffffba2e1977d12f3b7469971f30d28b94bd8  whole_whole2_partial
+
+
+class TestTreeHasher(unittest2.TestCase):
+
+
+
+    def test_treehasher(self):
+
+        shelltest("rm -rf /tmp/treehashertest; mkdir /tmp/treehashertest")
+        shelltest("cp tests/data/whole /tmp/treehashertest")
+        shelltest("mkdir /tmp/treehashertest/emptydir")
+        shelltest("mkdir /tmp/treehashertest/dir")
+        shelltest("cp tests/data/whole_whole2_partial /tmp/treehashertest/dir")
+
+        #it should ignore these:
+        shelltest("ln -s / /tmp/treehashertest/symlink")
+        shelltest("mknod /tmp/treehashertest/c c 1 1")
+        shelltest("mknod /tmp/treehashertest/b b 1 1")
+        shelltest("mkfifo /tmp/treehashertest/f")
+
+        block_hasher=BlockHasher(count=2)
+        tree_hasher=TreeHasher(block_hasher)
+
+        with self.subTest("Test output"):
+            self.assertEqual(list(tree_hasher.generate("/tmp/treehashertest")),[
+                ('whole', 0, '3c0bf91170d873b8e327d3bafb6bc074580d11b7'),
+                ('dir/whole_whole2_partial', 0, '959e6b58078f0cfd2fb3d37e978fda51820473ff'),
+                ('dir/whole_whole2_partial', 1, '642027d63bb0afd7e0ba197f2c66ad03e3d70de1')
+            ])
+
+        with self.subTest("Test compare"):
+            generator=tree_hasher.generate("/tmp/treehashertest")
+            count=tree_hasher.compare("/tmp/treehashertest", generator)
+            self.assertEqual(count,2)
+
+        with self.subTest("Test mismatch"):
+            generator=list(tree_hasher.generate("/tmp/treehashertest"))
+            shelltest("cp tests/data/whole2 /tmp/treehashertest/whole")
+
+            with self.assertRaisesRegex(Exception,"mismatch"):
+                tree_hasher.compare("/tmp/treehashertest", generator)
+
+        with self.subTest("Test missig file compare"):
+            generator=list(tree_hasher.generate("/tmp/treehashertest"))
+            shelltest("rm /tmp/treehashertest/whole")
+
+            with self.assertRaises(Exception):
+                tree_hasher.compare("/tmp/treehashertest", generator)

zfs_autobackup/BlockHasher.py

@@ -44,7 +44,7 @@ class BlockHasher():
         checked=0
         with open(fname, "rb") as f:
             for ( chunk_nr, hexdigest ) in generator:
-                print ("comparing {} {} {}".format(fname, chunk_nr, hexdigest))
+                # print ("comparing {} {} {}".format(fname, chunk_nr, hexdigest))
 
                 checked=checked+1
                 hash = self.hash_class()

zfs_autobackup/TreeHasher.py

@@ -43,6 +43,7 @@ class TreeHasher():
 
         cwd=os.getcwd()
         os.chdir(start_path)
+        count=0
         try:
 
             def filter_file_name( file_name, chunk_nr, hexdigest):
@@ -50,10 +51,13 @@ class TreeHasher():
 
 
             for file_name, group_generator in itertools.groupby(generator, lambda x: x[0]):
+                count=count+1
                 block_generator=itertools.starmap(filter_file_name, group_generator)
                 self.block_hasher.compare(file_name, block_generator)
         finally:
             os.chdir(cwd)
 
+        return count
+