From 1cedea5f5f4db226f45e849c79727aefb037ae10 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 23 Feb 2022 21:31:00 +0100 Subject: [PATCH] zfscheck wip --- zfs_autobackup/BlockHasher.py | 6 ++++- zfs_autobackup/ZfsCheck.py | 49 ++++++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/zfs_autobackup/BlockHasher.py b/zfs_autobackup/BlockHasher.py index c891b44..255aadd 100644 --- a/zfs_autobackup/BlockHasher.py +++ b/zfs_autobackup/BlockHasher.py @@ -12,10 +12,14 @@ class BlockHasher(): Input and output generators are in the format ( chunk_nr, hexdigest ) """ - def __init__(self, count=10000, bs=4096, hash_class=hashlib.sha1): + def __init__(self, count=10000, bs=4096, hash_class=hashlib.sha1, coverage=1): self.count = count self.bs = bs self.hash_class = hash_class + self.coverage=1 + + self.stats_total=0 + self.stats_checked=0 def generate(self, fname): """Generates checksums diff --git a/zfs_autobackup/ZfsCheck.py b/zfs_autobackup/ZfsCheck.py index 9166b7e..0754837 100644 --- a/zfs_autobackup/ZfsCheck.py +++ b/zfs_autobackup/ZfsCheck.py @@ -1,6 +1,7 @@ from __future__ import print_function import time +from random import random, randint from signal import signal, SIGPIPE from .TreeHasher import TreeHasher @@ -18,27 +19,32 @@ class ZfsCheck(CliBase): super(ZfsCheck, self).__init__(argv, print_arguments) self.node = ZfsNode(self.log, readonly=self.args.test, debug_output=self.args.debug_output) - self.block_hasher = BlockHasher(count=self.args.count, bs=self.args.block_size) + + if self.args.check is None: + self.block_hasher = BlockHasher(count=self.args.count, bs=self.args.block_size) + else: + self.block_hasher = BlockHasher(count=self.args.count, bs=self.args.block_size, coverage=self.args.percentage) def get_parser(self): parser = super(ZfsCheck, self).get_parser() # positional arguments - parser.add_argument('target', metavar='TARGET', default=None, nargs='?', help='Target to checksum. (can be blockdevice, directory or ZFS snapshot)') + parser.add_argument('target', metavar='TARGET', default=None, nargs='?', help='Target to checkum. (can be blockdevice, directory or ZFS snapshot)') - group = parser.add_argument_group('Hasher options') + group = parser.add_argument_group('Checker options') group.add_argument('--block-size', metavar="BYTES", default=4096, help="Read block-size, default %(default)s", type=int) group.add_argument('--count', metavar="COUNT", default=int((100 * (1024 ** 2)) / 4096), help="Hash chunks of COUNT blocks. Default %(default)s . (Chunk size is BYTES * COUNT) ", type=int) # 100MiB - group = parser.add_argument_group('Compare options') - group.add_argument('--check', '-c', metavar="FILE", default=None, const=True, nargs='?', help="Read hashes from STDIN (or FILE) and compare them") + group.add_argument('--percentage', '-p', metavar="NUMBER", default=100, type=float, + help="Generate/compare only this percentage of hashes. Default %(default)s") + return parser def parse_args(self, argv): @@ -51,6 +57,15 @@ class ZfsCheck(CliBase): self.error("Please specify TARGET") sys.exit(1) + self.verbose("Target : {}".format(args.target)) + self.verbose("Block size : {} bytes".format(args.block_size)) + self.verbose("Block count : {}".format(args.count)) + self.verbose("Effective chunk size : {} bytes".format(args.count*args.block_size)) + self.verbose("Percentage to check : {} %".format(args.percentage)) + self.verbose("") + + args.percentage=args.percentage/100 + return args def generate_zfs_filesystem(self, snapshot, input_generator): @@ -180,13 +195,13 @@ class ZfsCheck(CliBase): """generate checksums or compare (and generate error messages)""" if '@' in self.args.target: - self.verbose("Assuming target {} is ZFS snapshot.".format(self.args.target)) + self.verbose("Target is a ZFS snapshot.".format(self.args.target)) return self.generate_zfs_target(input_generator) elif os.path.isdir(self.args.target): - self.verbose("Target {} is directory, checking recursively.".format(self.args.target)) + self.verbose("Target is a directory, checking recursively.".format(self.args.target)) return self.generate_path(input_generator) elif os.path.exists(self.args.target): - self.verbose("Target {} is single file or blockdevice.".format(self.args.target)) + self.verbose("Target is single file or blockdevice.".format(self.args.target)) return self.generate_file(input_generator) else: raise Exception("Cant open {} ".format(self.args.target)) @@ -200,7 +215,8 @@ class ZfsCheck(CliBase): input_fh=open(file_name, 'r') last_progress_time = time.time() - progress_count = 0 + progress_checked = 0 + progress_total = 0 line=input_fh.readline() while line: @@ -208,15 +224,21 @@ class ZfsCheck(CliBase): #ignores lines without tabs if (len(i)>1): - yield i + if self.args.percentage==1 or self.args.percentage>random(): + progress_checked=progress_checked+1 + yield i + + progress_total=progress_total+1 - progress_count=progress_count+1 if self.args.progress and time.time() - last_progress_time > 1: last_progress_time = time.time() - self.progress("Checked {} hashes.".format(progress_count)) + self.progress("Checked {}/{} hashes. ({:.2f}% coverage)".format(progress_checked, progress_total, (float(progress_checked)/progress_total)*100)) line=input_fh.readline() + self.verbose("Checked {}/{} hashes. ({:.2f}% coverage)".format(progress_checked, progress_total, ( + float(progress_checked) / progress_total) * 100)) + def run(self): try: @@ -239,6 +261,7 @@ class ZfsCheck(CliBase): sys.stdout.flush() + self.verbose("Generated {} hashes.".format(progress_count)) self.clear_progress() return 0 @@ -258,6 +281,8 @@ class ZfsCheck(CliBase): sys.stdout.flush() + self.verbose("Total errors: {}".format(errors)) + self.clear_progress() return min(255,errors)