diff --git a/tests/test_blockhasher.py b/tests/test_blockhasher.py new file mode 100644 index 0000000..e8725ea --- /dev/null +++ b/tests/test_blockhasher.py @@ -0,0 +1,100 @@ +from basetest import * +from zfs_autobackup.BlockHasher import BlockHasher + +# make VERY sure this works correctly under all circumstances. + +# sha1 sums of files, (bs=4096) +# da39a3ee5e6b4b0d3255bfef95601890afd80709 empty +# 642027d63bb0afd7e0ba197f2c66ad03e3d70de1 partial +# 3c0bf91170d873b8e327d3bafb6bc074580d11b7 whole +# 2e863f1fcccd6642e4e28453eba10d2d3f74d798 whole2 +# 959e6b58078f0cfd2fb3d37e978fda51820473ff whole_whole2 +# 309ffffba2e1977d12f3b7469971f30d28b94bd8 whole_whole2_partial + +class TestBlockHahser(unittest2.TestCase): + + def setUp(self): + pass + + def test_empty(self): + block_hasher=BlockHasher(count=1) + self.assertEqual( + list(block_hasher.generate("tests/data/empty")), + [] + ) + + def test_partial(self): + block_hasher = BlockHasher(count=1) + self.assertEqual( + list(block_hasher.generate("tests/data/partial")), + [(0, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1")] + ) + + def test_whole(self): + block_hasher = BlockHasher(count=1) + self.assertEqual( + list(block_hasher.generate("tests/data/whole")), + [(0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7")] + ) + + def test_whole2(self): + block_hasher = BlockHasher(count=1) + self.assertEqual( + list(block_hasher.generate("tests/data/whole_whole2")), + [ + (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), + (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798") + ] + ) + + def test_wwp(self): + block_hasher = BlockHasher(count=1) + self.assertEqual( + list(block_hasher.generate("tests/data/whole_whole2_partial")), + [ + (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), #whole + (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), #whole2 + (2, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1") #partial + ] + ) + + def test_wwp_count2(self): + block_hasher = BlockHasher(count=2) + self.assertEqual( + list(block_hasher.generate("tests/data/whole_whole2_partial")), + [ + (0, "959e6b58078f0cfd2fb3d37e978fda51820473ff"), #whole_whole2 + (1, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1") #partial + ] + ) + + def test_big(self): + block_hasher=BlockHasher(count=10) + self.assertEqual( + list(block_hasher.generate("tests/data/whole_whole2_partial")), + [ + (0, "309ffffba2e1977d12f3b7469971f30d28b94bd8"), #whole_whole2_partial + ]) + + def test_blockhash_compare(self): + + + block_hasher=BlockHasher(count=1) + generator=block_hasher.generate("tests/data/whole_whole2_partial") + self.assertEqual(3,block_hasher.compare("tests/data/whole_whole2_partial", generator)) + + block_hasher=BlockHasher(count=1) + with self.assertRaisesRegexp(Exception, "^Block 1 mismatched!"): + generator=block_hasher.generate("tests/data/whole_whole2_partial") + self.assertEqual(3,block_hasher.compare("tests/data/whole", generator)) + + block_hasher=BlockHasher(count=10) + generator=block_hasher.generate("tests/data/whole_whole2_partial") + self.assertEqual(1,block_hasher.compare("tests/data/whole_whole2_partial", generator)) + + #different order to make sure seek functions + block_hasher=BlockHasher(count=1) + checksums=list(block_hasher.generate("tests/data/whole_whole2_partial")) + checksums.reverse() + self.assertEqual(3,block_hasher.compare("tests/data/whole_whole2_partial", checksums)) + diff --git a/tests/test_treehasher.py b/tests/test_treehasher.py new file mode 100644 index 0000000..888df2c --- /dev/null +++ b/tests/test_treehasher.py @@ -0,0 +1,58 @@ +from basetest import * +from zfs_autobackup.BlockHasher import BlockHasher + +# sha1 sums of files, (bs=4096) +# da39a3ee5e6b4b0d3255bfef95601890afd80709 empty +# 642027d63bb0afd7e0ba197f2c66ad03e3d70de1 partial +# 3c0bf91170d873b8e327d3bafb6bc074580d11b7 whole +# 2e863f1fcccd6642e4e28453eba10d2d3f74d798 whole2 +# 959e6b58078f0cfd2fb3d37e978fda51820473ff whole_whole2 +# 309ffffba2e1977d12f3b7469971f30d28b94bd8 whole_whole2_partial + + +class TestTreeHasher(unittest2.TestCase): + + + + def test_treehasher(self): + + shelltest("rm -rf /tmp/treehashertest; mkdir /tmp/treehashertest") + shelltest("cp tests/data/whole /tmp/treehashertest") + shelltest("mkdir /tmp/treehashertest/emptydir") + shelltest("mkdir /tmp/treehashertest/dir") + shelltest("cp tests/data/whole_whole2_partial /tmp/treehashertest/dir") + + #it should ignore these: + shelltest("ln -s / /tmp/treehashertest/symlink") + shelltest("mknod /tmp/treehashertest/c c 1 1") + shelltest("mknod /tmp/treehashertest/b b 1 1") + shelltest("mkfifo /tmp/treehashertest/f") + + block_hasher=BlockHasher(count=2) + tree_hasher=TreeHasher(block_hasher) + + with self.subTest("Test output"): + self.assertEqual(list(tree_hasher.generate("/tmp/treehashertest")),[ + ('whole', 0, '3c0bf91170d873b8e327d3bafb6bc074580d11b7'), + ('dir/whole_whole2_partial', 0, '959e6b58078f0cfd2fb3d37e978fda51820473ff'), + ('dir/whole_whole2_partial', 1, '642027d63bb0afd7e0ba197f2c66ad03e3d70de1') + ]) + + with self.subTest("Test compare"): + generator=tree_hasher.generate("/tmp/treehashertest") + count=tree_hasher.compare("/tmp/treehashertest", generator) + self.assertEqual(count,2) + + with self.subTest("Test mismatch"): + generator=list(tree_hasher.generate("/tmp/treehashertest")) + shelltest("cp tests/data/whole2 /tmp/treehashertest/whole") + + with self.assertRaisesRegex(Exception,"mismatch"): + tree_hasher.compare("/tmp/treehashertest", generator) + + with self.subTest("Test missig file compare"): + generator=list(tree_hasher.generate("/tmp/treehashertest")) + shelltest("rm /tmp/treehashertest/whole") + + with self.assertRaises(Exception): + tree_hasher.compare("/tmp/treehashertest", generator) diff --git a/zfs_autobackup/BlockHasher.py b/zfs_autobackup/BlockHasher.py index 22e2b61..faf596f 100644 --- a/zfs_autobackup/BlockHasher.py +++ b/zfs_autobackup/BlockHasher.py @@ -44,7 +44,7 @@ class BlockHasher(): checked=0 with open(fname, "rb") as f: for ( chunk_nr, hexdigest ) in generator: - print ("comparing {} {} {}".format(fname, chunk_nr, hexdigest)) + # print ("comparing {} {} {}".format(fname, chunk_nr, hexdigest)) checked=checked+1 hash = self.hash_class() diff --git a/zfs_autobackup/TreeHasher.py b/zfs_autobackup/TreeHasher.py index c626aee..54bec1e 100644 --- a/zfs_autobackup/TreeHasher.py +++ b/zfs_autobackup/TreeHasher.py @@ -43,6 +43,7 @@ class TreeHasher(): cwd=os.getcwd() os.chdir(start_path) + count=0 try: def filter_file_name( file_name, chunk_nr, hexdigest): @@ -50,10 +51,13 @@ class TreeHasher(): for file_name, group_generator in itertools.groupby(generator, lambda x: x[0]): + count=count+1 block_generator=itertools.starmap(filter_file_name, group_generator) self.block_hasher.compare(file_name, block_generator) finally: os.chdir(cwd) + return count +