diff --git a/tests/run_tests b/tests/run_tests index 87d43fa..b2347d4 100755 --- a/tests/run_tests +++ b/tests/run_tests @@ -18,6 +18,7 @@ if ! [ -e /root/.ssh/id_rsa ]; then ssh -oStrictHostKeyChecking=no localhost true || exit 1 fi +umount /tmp/ZfsCheck* coverage run --branch --source zfs_autobackup -m unittest discover -vvvvf $SCRIPTDIR $@ 2>&1 EXIT=$? diff --git a/tests/test_zfscheck.py b/tests/test_zfscheck.py index 4f75b59..8a24973 100644 --- a/tests/test_zfscheck.py +++ b/tests/test_zfscheck.py @@ -181,7 +181,7 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bd8 #breaks pipe when grep exists: #important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling shelltest("python -m zfs_autobackup.ZfsCheck test_source1@test --debug | grep -m1 'Hashing tree'") - time.sleep(1) + # time.sleep(5) #should NOT be mounted anymore if cleanup went ok: self.assertNotRegex(shelltest("mount"), "test_source1@test") @@ -194,11 +194,11 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bd8 #breaks pipe when grep exists: #important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling - shelltest("python -m zfs_autobackup.ZfsCheck test_source1/vol@test --debug | grep -m1 'Hashing dev'") - time.sleep(1) + shelltest("python -m zfs_autobackup.ZfsCheck test_source1/vol@test --debug | grep -m1 'Hashing file'") + # time.sleep(1) r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) - self.assertMultiLineEqual(r, """ + self.assertMultiLineEqual(""" test_source1 test_source1/fs1 test_source1/fs1/sub @@ -210,7 +210,7 @@ test_source2/fs2/sub test_source2/fs3 test_source2/fs3/sub test_target1 -""") +""",r ) diff --git a/zfs_autobackup/BlockHasher.py b/zfs_autobackup/BlockHasher.py index 3ea4252..0a37c90 100644 --- a/zfs_autobackup/BlockHasher.py +++ b/zfs_autobackup/BlockHasher.py @@ -70,11 +70,11 @@ class BlockHasher(): with open(fname, "rb") as fh: - fsize = fh.seek(0, os.SEEK_END) + fh.seek(0, os.SEEK_END) + fsize=fh.tell() fh.seek(0) while fh.tell() 1: + last_progress_time = time.time() + self.progress("Generated {} hashes.".format(progress_count)) + + sys.stdout.flush() + + self.verbose("Generated {} hashes.".format(progress_count)) + self.clear_progress() + + return 0 + + def print_errors(self, compare_generator): + """prints errors that are yielded by the specified compare_generator""" + errors = 0 + for i in compare_generator: + errors = errors + 1 + + if len(i) == 4: + (file_name, chunk_nr, compare_hexdigest, actual_hexdigest) = i + print("{}: Chunk {} failed: {} {}".format(file_name, chunk_nr, compare_hexdigest, actual_hexdigest)) + else: + (chunk_nr, compare_hexdigest, actual_hexdigest) = i + print("Chunk {} failed: {} {}".format(chunk_nr, compare_hexdigest, actual_hexdigest)) + + sys.stdout.flush() + + self.verbose("Total errors: {}".format(errors)) + self.clear_progress() + + return errors + + def prepare_target(self): + + if "@" in self.args.target: + # zfs snapshot + snapshot=self.node.get_dataset(self.args.target) + dataset_type = snapshot.parent.properties['type'] + + if dataset_type == 'volume': + return self.prepare_zfs_volume(snapshot) + elif dataset_type == 'filesystem': + return self.prepare_zfs_filesystem(snapshot) + else: + raise Exception("Unknown dataset type") + return self.args.target + + def cleanup_target(self): + if "@" in self.args.target: + # zfs snapshot + snapshot=self.node.get_dataset(self.args.target) + dataset_type = snapshot.parent.properties['type'] + + if dataset_type == 'volume': + self.cleanup_zfs_volume(snapshot) + elif dataset_type == 'filesystem': + self.cleanup_zfs_filesystem(snapshot) + def run(self): + compare_generator=None + hash_generator=None try: - last_progress_time = time.time() - progress_count = 0 - - #run as generator - if self.args.check==None: - for i in self.generate(input_generator=None): - - if len(i)==3: - print("{}\t{}\t{}".format(*i)) - else: - print("{}\t{}".format(*i)) - progress_count=progress_count+1 - - if self.args.progress and time.time()-last_progress_time>1: - last_progress_time=time.time() - self.progress("Generated {} hashes.".format(progress_count)) - - sys.stdout.flush() - - self.verbose("Generated {} hashes.".format(progress_count)) - self.clear_progress() - return 0 + prepared_target=self.prepare_target() + is_dir=os.path.isdir(prepared_target) #run as compare + if self.args.check is not None: + input_generator=self.generate_input() + if is_dir: + compare_generator = self.generate_tree_compare(prepared_target, input_generator) + else: + compare_generator=self.generate_file_compare(prepared_target, input_generator) + errors=self.print_errors(compare_generator) + #run as generator else: - errors=0 - input_generator=self.input_parser(self.args.check) - for i in self.generate(input_generator): - errors=errors+1 + if is_dir: + hash_generator = self.generate_tree_hashes(prepared_target) + else: + hash_generator=self.generate_file_hashes(prepared_target) - if len(i)==4: - (file_name, chunk_nr, compare_hexdigest, actual_hexdigest)=i - print("{}: Chunk {} failed: {} {}".format(file_name, chunk_nr, compare_hexdigest, actual_hexdigest)) - else: - (chunk_nr, compare_hexdigest, actual_hexdigest) = i - print("Chunk {} failed: {} {}".format(chunk_nr, compare_hexdigest, actual_hexdigest)) - - sys.stdout.flush() - - self.verbose("Total errors: {}".format(errors)) - - self.clear_progress() - return min(255,errors) + errors=self.print_hashes(hash_generator) except Exception as e: self.error("Exception: " + str(e)) @@ -292,12 +280,26 @@ class ZfsCheck(CliBase): self.error("Aborted") return 255 + finally: + #important to call check_output so that cleanup still functions in case of a broken pipe: + # util.check_output() + + #close generators, to make sure files are not in use anymore when cleaning up + if hash_generator is not None: + hash_generator.close() + if compare_generator is not None: + compare_generator.close() + self.cleanup_target() + + return errors + + def cli(): import sys signal(SIGPIPE, sigpipe_handler) sys.exit(ZfsCheck(sys.argv[1:], False).run()) -if __name__ == "__main__": +if __name__ == "__main__": cli() diff --git a/zfs_autobackup/test.py b/zfs_autobackup/test.py index 6c1e39b..14b6bba 100644 --- a/zfs_autobackup/test.py +++ b/zfs_autobackup/test.py @@ -1,70 +1,129 @@ import os.path import os +import subprocess +import sys import time -from random import random +from signal import signal, SIGPIPE -with open('test.py', 'rb') as fh: +import util - # fsize = fh.seek(10000, os.SEEK_END) - # print(fsize) - - start=time.time() - for i in range(0,1000000): - # fh.seek(0, 0) - fsize=fh.seek(0, os.SEEK_END) - # fsize=fh.tell() - # os.path.getsize('test.py') - print(time.time()-start) +signal(SIGPIPE, util.sigpipe_handler) - print(fh.tell()) +try: + print ("voor eerste") + raise Exception("eerstre") +except Exception as e: + print ("voor tweede") + raise Exception("tweede") +finally: + print ("JO") -sys.exit(0) +def generator(): + + try: + util.deb('in generator') + print ("TRIGGER SIGPIPE") + sys.stdout.flush() + util.deb('after trigger') + + # if False: + yield ("bla") + # yield ("bla") + + except GeneratorExit as e: + util.deb('GENEXIT '+str(e)) + raise + + except Exception as e: + util.deb('EXCEPT '+str(e)) + finally: + util.deb('FINALLY') + print("nog iets") + sys.stdout.flush() + util.deb('after print in finally WOOP!') +util.deb('START') +g=generator() +util.deb('after generator') +for bla in g: + # print ("heb wat ontvangen") + util.deb('ontvangen van gen') + break + # raise Exception("moi") -checked=1 -skipped=1 -coverage=0.1 + pass +raise Exception("moi") -max_skip=0 +util.deb('after for') - -skipinarow=0 while True: - total=checked+skipped + pass - skip=coveragemax_skip: - max_skip=skipinarow - else: - skipinarow=0 - checked=checked+1 - print("C {:.2f}%".format(checked * 100 / total)) - - print(max_skip) - -skip=0 -while True: - - total=checked+skipped - if skip>0: - skip=skip-1 - skipped = skipped + 1 - print("S {:.2f}%".format(checked * 100 / total)) - else: - checked=checked+1 - print("C {:.2f}%".format(checked * 100 / total)) - - #calc new skip - skip=skip+((1/coverage)-1)*(random()*2) - # print(skip) - if skip> max_skip: - max_skip=skip - - print(max_skip) +# +# with open('test.py', 'rb') as fh: +# +# # fsize = fh.seek(10000, os.SEEK_END) +# # print(fsize) +# +# start=time.time() +# for i in range(0,1000000): +# # fh.seek(0, 0) +# fsize=fh.seek(0, os.SEEK_END) +# # fsize=fh.tell() +# # os.path.getsize('test.py') +# print(time.time()-start) +# +# +# print(fh.tell()) +# +# sys.exit(0) +# +# +# +# checked=1 +# skipped=1 +# coverage=0.1 +# +# max_skip=0 +# +# +# skipinarow=0 +# while True: +# total=checked+skipped +# +# skip=coveragemax_skip: +# max_skip=skipinarow +# else: +# skipinarow=0 +# checked=checked+1 +# print("C {:.2f}%".format(checked * 100 / total)) +# +# print(max_skip) +# +# skip=0 +# while True: +# +# total=checked+skipped +# if skip>0: +# skip=skip-1 +# skipped = skipped + 1 +# print("S {:.2f}%".format(checked * 100 / total)) +# else: +# checked=checked+1 +# print("C {:.2f}%".format(checked * 100 / total)) +# +# #calc new skip +# skip=skip+((1/coverage)-1)*(random()*2) +# # print(skip) +# if skip> max_skip: +# max_skip=skip +# +# print(max_skip) diff --git a/zfs_autobackup/util.py b/zfs_autobackup/util.py index 5c86041..4d774b3 100644 --- a/zfs_autobackup/util.py +++ b/zfs_autobackup/util.py @@ -19,7 +19,7 @@ import sys def tmp_name(suffix=""): - """create temporary name unique to this process and node""" + """create temporary name unique to this process and node. always retruns the same result during the same execution""" #we could use uuids but those are ugly and confusing name="{}-{}-{}".format( @@ -48,3 +48,18 @@ def output_redir(): def sigpipe_handler(sig, stack): #redir output so we dont get more SIGPIPES during cleanup. (which my try to write to stdout) output_redir() + deb('redir') + +# def check_output(): +# """make sure stdout still functions. if its broken, this will trigger a SIGPIPE which will be handled by the sigpipe_handler.""" +# try: +# print(" ") +# sys.stdout.flush() +# except Exception as e: +# pass + +# def deb(txt): +# with open('/tmp/debug.log', 'a') as fh: +# fh.write("DEB: "+txt+"\n") + +