norohind/zfs_autobackup
1
0
Fork 0
forked from third-party-mirrors/zfs_autobackup
Code Pull Requests Activity

now yields errors and mismatches

Browse Source
This commit is contained in:
Edwin Eefting 2022-02-22 14:47:15 +01:00
parent 3f755fcc69
commit 6a58a294a3
4 changed files with 74 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
tests/test_blockhasher.py
View File

@ -1,6 +1,7 @@
from basetest import * from basetest import *
from zfs_autobackup.BlockHasher import BlockHasher from zfs_autobackup.BlockHasher import BlockHasher
# make VERY sure this works correctly under all circumstances. # make VERY sure this works correctly under all circumstances.
# sha1 sums of files, (bs=4096) # sha1 sums of files, (bs=4096)
@ -11,13 +12,13 @@ from zfs_autobackup.BlockHasher import BlockHasher
# 959e6b58078f0cfd2fb3d37e978fda51820473ff whole_whole2 # 959e6b58078f0cfd2fb3d37e978fda51820473ff whole_whole2
# 309ffffba2e1977d12f3b7469971f30d28b94bd8 whole_whole2_partial # 309ffffba2e1977d12f3b7469971f30d28b94bd8 whole_whole2_partial
class TestBlockHahser(unittest2.TestCase): class TestBlockHasher(unittest2.TestCase):
def setUp(self): def setUp(self):
pass pass
def test_empty(self): def test_empty(self):
block_hasher=BlockHasher(count=1) block_hasher = BlockHasher(count=1)
self.assertEqual( self.assertEqual(
list(block_hasher.generate("tests/data/empty")), list(block_hasher.generate("tests/data/empty")),
[] []
@ -52,9 +53,9 @@ class TestBlockHahser(unittest2.TestCase):
self.assertEqual( self.assertEqual(
list(block_hasher.generate("tests/data/whole_whole2_partial")), list(block_hasher.generate("tests/data/whole_whole2_partial")),
[ [
(0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), #whole (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), # whole
(1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), #whole2 (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), # whole2
(2, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1") #partial (2, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1") # partial
] ]
) )
@ -63,38 +64,37 @@ class TestBlockHahser(unittest2.TestCase):
self.assertEqual( self.assertEqual(
list(block_hasher.generate("tests/data/whole_whole2_partial")), list(block_hasher.generate("tests/data/whole_whole2_partial")),
[ [
(0, "959e6b58078f0cfd2fb3d37e978fda51820473ff"), #whole_whole2 (0, "959e6b58078f0cfd2fb3d37e978fda51820473ff"), # whole_whole2
(1, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1") #partial (1, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1") # partial
] ]
) )
def test_big(self): def test_big(self):
block_hasher=BlockHasher(count=10) block_hasher = BlockHasher(count=10)
self.assertEqual( self.assertEqual(
list(block_hasher.generate("tests/data/whole_whole2_partial")), list(block_hasher.generate("tests/data/whole_whole2_partial")),
[ [
(0, "309ffffba2e1977d12f3b7469971f30d28b94bd8"), #whole_whole2_partial (0, "309ffffba2e1977d12f3b7469971f30d28b94bd8"), # whole_whole2_partial
]) ])
def test_blockhash_compare(self): def test_blockhash_compare(self):
block_hasher = BlockHasher(count=1)
generator = block_hasher.generate("tests/data/whole_whole2_partial")
self.assertEqual([], list(block_hasher.compare("tests/data/whole_whole2_partial", generator)))
block_hasher = BlockHasher(count=1)
generator = block_hasher.generate("tests/data/whole_whole2_partial")
self.assertEqual(
[(1, '2e863f1fcccd6642e4e28453eba10d2d3f74d798', 'EOF'),
(2, '642027d63bb0afd7e0ba197f2c66ad03e3d70de1', 'EOF')],
list(block_hasher.compare("tests/data/whole", generator)))
block_hasher=BlockHasher(count=1) block_hasher = BlockHasher(count=10)
generator=block_hasher.generate("tests/data/whole_whole2_partial") generator = block_hasher.generate("tests/data/whole_whole2_partial")
self.assertEqual(3,block_hasher.compare("tests/data/whole_whole2_partial", generator)) self.assertEqual([], list(block_hasher.compare("tests/data/whole_whole2_partial", generator)))
block_hasher=BlockHasher(count=1) # different order to make sure seek functions
with self.assertRaisesRegexp(Exception, "^Block 1 mismatched!"): block_hasher = BlockHasher(count=1)
generator=block_hasher.generate("tests/data/whole_whole2_partial") checksums = list(block_hasher.generate("tests/data/whole_whole2_partial"))
self.assertEqual(3,block_hasher.compare("tests/data/whole", generator))
block_hasher=BlockHasher(count=10)
generator=block_hasher.generate("tests/data/whole_whole2_partial")
self.assertEqual(1,block_hasher.compare("tests/data/whole_whole2_partial", generator))
#different order to make sure seek functions
block_hasher=BlockHasher(count=1)
checksums=list(block_hasher.generate("tests/data/whole_whole2_partial"))
checksums.reverse() checksums.reverse()
self.assertEqual(3,block_hasher.compare("tests/data/whole_whole2_partial", checksums)) self.assertEqual([], list(block_hasher.compare("tests/data/whole_whole2_partial", checksums)))

35
tests/test_treehasher.py
View File

@ -1,6 +1,7 @@
from basetest import * from basetest import *
from zfs_autobackup.BlockHasher import BlockHasher from zfs_autobackup.BlockHasher import BlockHasher
# sha1 sums of files, (bs=4096) # sha1 sums of files, (bs=4096)
# da39a3ee5e6b4b0d3255bfef95601890afd80709 empty # da39a3ee5e6b4b0d3255bfef95601890afd80709 empty
# 642027d63bb0afd7e0ba197f2c66ad03e3d70de1 partial # 642027d63bb0afd7e0ba197f2c66ad03e3d70de1 partial
@ -12,47 +13,47 @@ from zfs_autobackup.BlockHasher import BlockHasher
class TestTreeHasher(unittest2.TestCase): class TestTreeHasher(unittest2.TestCase):
def test_treehasher(self): def test_treehasher(self):
shelltest("rm -rf /tmp/treehashertest; mkdir /tmp/treehashertest") shelltest("rm -rf /tmp/treehashertest; mkdir /tmp/treehashertest")
shelltest("cp tests/data/whole /tmp/treehashertest") shelltest("cp tests/data/whole /tmp/treehashertest")
shelltest("mkdir /tmp/treehashertest/emptydir") shelltest("mkdir /tmp/treehashertest/emptydir")
shelltest("mkdir /tmp/treehashertest/dir") shelltest("mkdir /tmp/treehashertest/dir")
shelltest("cp tests/data/whole_whole2_partial /tmp/treehashertest/dir") shelltest("cp tests/data/whole_whole2_partial /tmp/treehashertest/dir")
#it should ignore these: # it should ignore these:
shelltest("ln -s / /tmp/treehashertest/symlink") shelltest("ln -s / /tmp/treehashertest/symlink")
shelltest("mknod /tmp/treehashertest/c c 1 1") shelltest("mknod /tmp/treehashertest/c c 1 1")
shelltest("mknod /tmp/treehashertest/b b 1 1") shelltest("mknod /tmp/treehashertest/b b 1 1")
shelltest("mkfifo /tmp/treehashertest/f") shelltest("mkfifo /tmp/treehashertest/f")
block_hasher=BlockHasher(count=2) block_hasher = BlockHasher(count=2)
tree_hasher=TreeHasher(block_hasher) tree_hasher = TreeHasher(block_hasher)
with self.subTest("Test output"): with self.subTest("Test output"):
self.assertEqual(list(tree_hasher.generate("/tmp/treehashertest")),[ self.assertEqual(list(tree_hasher.generate("/tmp/treehashertest")), [
('whole', 0, '3c0bf91170d873b8e327d3bafb6bc074580d11b7'), ('whole', 0, '3c0bf91170d873b8e327d3bafb6bc074580d11b7'),
('dir/whole_whole2_partial', 0, '959e6b58078f0cfd2fb3d37e978fda51820473ff'), ('dir/whole_whole2_partial', 0, '959e6b58078f0cfd2fb3d37e978fda51820473ff'),
('dir/whole_whole2_partial', 1, '642027d63bb0afd7e0ba197f2c66ad03e3d70de1') ('dir/whole_whole2_partial', 1, '642027d63bb0afd7e0ba197f2c66ad03e3d70de1')
]) ])
with self.subTest("Test compare"): with self.subTest("Test compare"):
generator=tree_hasher.generate("/tmp/treehashertest") generator = tree_hasher.generate("/tmp/treehashertest")
count=tree_hasher.compare("/tmp/treehashertest", generator) errors = list(tree_hasher.compare("/tmp/treehashertest", generator))
self.assertEqual(count,2) self.assertEqual(errors, [])
with self.subTest("Test mismatch"): with self.subTest("Test mismatch"):
generator=list(tree_hasher.generate("/tmp/treehashertest")) generator = list(tree_hasher.generate("/tmp/treehashertest"))
shelltest("cp tests/data/whole2 /tmp/treehashertest/whole") shelltest("cp tests/data/whole2 /tmp/treehashertest/whole")
with self.assertRaisesRegex(Exception,"mismatch"): self.assertEqual(list(tree_hasher.compare("/tmp/treehashertest", generator)),
tree_hasher.compare("/tmp/treehashertest", generator) [('whole',
0,
'3c0bf91170d873b8e327d3bafb6bc074580d11b7',
'2e863f1fcccd6642e4e28453eba10d2d3f74d798')])
with self.subTest("Test missig file compare"): with self.subTest("Test missing file compare"):
generator=list(tree_hasher.generate("/tmp/treehashertest")) generator = list(tree_hasher.generate("/tmp/treehashertest"))
shelltest("rm /tmp/treehashertest/whole") shelltest("rm /tmp/treehashertest/whole")
with self.assertRaises(Exception): self.assertEqual(list(tree_hasher.compare("/tmp/treehashertest", generator)),
tree_hasher.compare("/tmp/treehashertest", generator) [('whole', '-', '-', "ERROR: [Errno 2] No such file or directory: 'whole'")])

49
zfs_autobackup/BlockHasher.py
View File

@ -9,11 +9,11 @@ class BlockHasher():
Its also possible to only read a certain percentage of blocks to just check a sample. Its also possible to only read a certain percentage of blocks to just check a sample.
""" """
def __init__(self, count=10000, bs=4096, hash_class=hashlib.sha1):
self.count=count
self.bs=bs
self.hash_class=hash_class
def __init__(self, count=10000, bs=4096, hash_class=hashlib.sha1):
self.count = count
self.bs = bs
self.hash_class = hash_class
def generate(self, fname): def generate(self, fname):
"""Generates checksums """Generates checksums
@ -39,25 +39,32 @@ class BlockHasher():
yield (chunk_nr, hash.hexdigest()) yield (chunk_nr, hash.hexdigest())
def compare(self, fname, generator): def compare(self, fname, generator):
"""reads from generator and compares blocks, raises exception on error""" """reads from generator and compares blocks, yields mismatches"""
checked=0 try:
with open(fname, "rb") as f: checked = 0
for ( chunk_nr, hexdigest ) in generator: with open(fname, "rb") as f:
# print ("comparing {} {} {}".format(fname, chunk_nr, hexdigest)) for (chunk_nr, hexdigest) in generator:
try:
checked=checked+1 checked = checked + 1
hash = self.hash_class() hash = self.hash_class()
f.seek(chunk_nr * self.bs * self.count) f.seek(chunk_nr * self.bs * self.count)
block_nr=0 block_nr = 0
for block in iter(lambda: f.read(self.bs), b""): for block in iter(lambda: f.read(self.bs), b""):
hash.update(block) hash.update(block)
block_nr=block_nr+1 block_nr = block_nr + 1
if block_nr == self.count: if block_nr == self.count:
break break
if (hash.hexdigest()!=hexdigest): if block_nr == 0:
raise Exception("Block {} mismatched! Hash is {}, but should be {}".format(chunk_nr, hash.hexdigest(), hexdigest)) yield (chunk_nr, hexdigest, 'EOF')
return checked elif (hash.hexdigest() != hexdigest):
yield (chunk_nr, hexdigest, hash.hexdigest())
except Exception as e:
yield ( chunk_nr , hexdigest, 'ERROR: '+str(e))
except Exception as e:
yield ( '-', '-', 'ERROR: '+ str(e))

4
zfs_autobackup/TreeHasher.py
View File

@ -53,11 +53,11 @@ class TreeHasher():
for file_name, group_generator in itertools.groupby(generator, lambda x: x[0]): for file_name, group_generator in itertools.groupby(generator, lambda x: x[0]):
count=count+1 count=count+1
block_generator=itertools.starmap(filter_file_name, group_generator) block_generator=itertools.starmap(filter_file_name, group_generator)
self.block_hasher.compare(file_name, block_generator) for ( chunk_nr, compare_hexdigest, actual_hexdigest) in self.block_hasher.compare(file_name, block_generator):
yield ( file_name, chunk_nr, compare_hexdigest, actual_hexdigest )
finally: finally:
os.chdir(cwd) os.chdir(cwd)
return count