third-party-mirrors/zfs_autobackup
1
0
Fork 1
mirror of https://github.com/psy0rz/zfs_autobackup.git synced 2025-04-11 22:40:01 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

zfscheck wip

Browse Source
This commit is contained in:
Edwin Eefting 2022-02-23 21:31:00 +01:00
parent d99c202e75
commit 1cedea5f5f
2 changed files with 42 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
zfs_autobackup/BlockHasher.py
View File

@ -12,10 +12,14 @@ class BlockHasher():
Input and output generators are in the format ( chunk_nr, hexdigest )
"""
def __init__(self, count=10000, bs=4096, hash_class=hashlib.sha1):
def __init__(self, count=10000, bs=4096, hash_class=hashlib.sha1, coverage=1):
self.count = count
self.bs = bs
self.hash_class = hash_class
self.coverage=1
self.stats_total=0
self.stats_checked=0
def generate(self, fname):
"""Generates checksums

49
zfs_autobackup/ZfsCheck.py
View File

@ -1,6 +1,7 @@
from __future__ import print_function
import time
from random import random, randint
from signal import signal, SIGPIPE
from .TreeHasher import TreeHasher
@ -18,27 +19,32 @@ class ZfsCheck(CliBase):
super(ZfsCheck, self).__init__(argv, print_arguments)
self.node = ZfsNode(self.log, readonly=self.args.test, debug_output=self.args.debug_output)
self.block_hasher = BlockHasher(count=self.args.count, bs=self.args.block_size)
if self.args.check is None:
self.block_hasher = BlockHasher(count=self.args.count, bs=self.args.block_size)
else:
self.block_hasher = BlockHasher(count=self.args.count, bs=self.args.block_size, coverage=self.args.percentage)
def get_parser(self):
parser = super(ZfsCheck, self).get_parser()
# positional arguments
parser.add_argument('target', metavar='TARGET', default=None, nargs='?', help='Target to checksum. (can be blockdevice, directory or ZFS snapshot)')
parser.add_argument('target', metavar='TARGET', default=None, nargs='?', help='Target to checkum. (can be blockdevice, directory or ZFS snapshot)')
group = parser.add_argument_group('Hasher options')
group = parser.add_argument_group('Checker options')
group.add_argument('--block-size', metavar="BYTES", default=4096, help="Read block-size, default %(default)s",
type=int)
group.add_argument('--count', metavar="COUNT", default=int((100 * (1024 ** 2)) / 4096),
help="Hash chunks of COUNT blocks. Default %(default)s . (Chunk size is BYTES * COUNT) ", type=int) # 100MiB
group = parser.add_argument_group('Compare options')
group.add_argument('--check', '-c', metavar="FILE", default=None, const=True, nargs='?',
help="Read hashes from STDIN (or FILE) and compare them")
group.add_argument('--percentage', '-p', metavar="NUMBER", default=100, type=float,
help="Generate/compare only this percentage of hashes. Default %(default)s")
return parser
def parse_args(self, argv):
@ -51,6 +57,15 @@ class ZfsCheck(CliBase):
self.error("Please specify TARGET")
sys.exit(1)
self.verbose("Target : {}".format(args.target))
self.verbose("Block size : {} bytes".format(args.block_size))
self.verbose("Block count : {}".format(args.count))
self.verbose("Effective chunk size : {} bytes".format(args.count*args.block_size))
self.verbose("Percentage to check : {} %".format(args.percentage))
self.verbose("")
args.percentage=args.percentage/100
return args
def generate_zfs_filesystem(self, snapshot, input_generator):
@ -180,13 +195,13 @@ class ZfsCheck(CliBase):
"""generate checksums or compare (and generate error messages)"""
if '@' in self.args.target:
self.verbose("Assuming target {} is ZFS snapshot.".format(self.args.target))
self.verbose("Target is a ZFS snapshot.".format(self.args.target))
return self.generate_zfs_target(input_generator)
elif os.path.isdir(self.args.target):
self.verbose("Target {} is directory, checking recursively.".format(self.args.target))
self.verbose("Target is a directory, checking recursively.".format(self.args.target))
return self.generate_path(input_generator)
elif os.path.exists(self.args.target):
self.verbose("Target {} is single file or blockdevice.".format(self.args.target))
self.verbose("Target is single file or blockdevice.".format(self.args.target))
return self.generate_file(input_generator)
else:
raise Exception("Cant open {} ".format(self.args.target))
@ -200,7 +215,8 @@ class ZfsCheck(CliBase):
input_fh=open(file_name, 'r')
last_progress_time = time.time()
progress_count = 0
progress_checked = 0
progress_total = 0
line=input_fh.readline()
while line:
@ -208,15 +224,21 @@ class ZfsCheck(CliBase):
#ignores lines without tabs
if (len(i)>1):
yield i
if self.args.percentage==1 or self.args.percentage>random():
progress_checked=progress_checked+1
yield i
progress_total=progress_total+1
progress_count=progress_count+1
if self.args.progress and time.time() - last_progress_time > 1:
last_progress_time = time.time()
self.progress("Checked {} hashes.".format(progress_count))
self.progress("Checked {}/{} hashes. ({:.2f}% coverage)".format(progress_checked, progress_total, (float(progress_checked)/progress_total)*100))
line=input_fh.readline()
self.verbose("Checked {}/{} hashes. ({:.2f}% coverage)".format(progress_checked, progress_total, (
float(progress_checked) / progress_total) * 100))
def run(self):
try:
@ -239,6 +261,7 @@ class ZfsCheck(CliBase):
sys.stdout.flush()
self.verbose("Generated {} hashes.".format(progress_count))
self.clear_progress()
return 0
@ -258,6 +281,8 @@ class ZfsCheck(CliBase):
sys.stdout.flush()
self.verbose("Total errors: {}".format(errors))
self.clear_progress()
return min(255,errors)