norohind/zfs_autobackup
1
0
Fork 0
forked from third-party-mirrors/zfs_autobackup
Code Pull Requests Activity

zfs-autoverify wip

Browse Source
This commit is contained in:
Edwin Eefting 2022-01-23 23:01:53 +01:00
parent 69975b37fb
commit f66957d867
3 changed files with 52 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
tests/test_verify.py
View File

@ -5,7 +5,8 @@ from basetest import *
# test zfs-verify:
# - when there is no common snapshot at all
# - when encryption key not loaded
# - test mode
# - --test mode
# - --fs-compare methods
# - on snapshots of datasets:
# - that are correct
# - that are different
@ -21,21 +22,41 @@ class TestZfsEncryption(unittest2.TestCase):
def setUp(self):
prepare_zpools()
#create actual test files and data
shelltest("zfs create test_source1/fs1/ok_filesystem")
shelltest("cp *.py /test_source1/fs1/ok_filesystem")
shelltest("cp tests/*.py /test_source1/fs1/ok_filesystem")
shelltest("zfs create test_source1/fs1/bad_filesystem")
shelltest("cp tests/*.py /test_source1/fs1/bad_filesystem")
shelltest("zfs create -V 1M test_source1/fs1/ok_zvol")
shelltest("dd if=/dev/urandom of=/dev/zvol/test_source1/fs1/ok_zvol count=1 bs=512k")
#create backup
with patch('time.strftime', return_value="test-20101111000000"):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress".split(" ")).run())
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --no-holds".split(" ")).run())
#Do an ugly hack to create a fault in the bad filesystem
#In zfs-autoverify it doenst matter that the snapshot isnt actually the same snapshot, so this hack works
shelltest("zfs destroy test_target1/test_source1/fs1/bad_filesystem@test-20101111000000")
shelltest("zfs mount test_target1/test_source1/fs1/bad_filesystem")
shelltest("echo >> /test_target1/test_source1/fs1/bad_filesystem/test_verify.py")
shelltest("zfs snapshot test_target1/test_source1/fs1/bad_filesystem@test-20101111000000")
# make sure we cant accidently compare current data
shelltest("zfs mount test_target1/test_source1/fs1/ok_filesystem")
shelltest("rm /test_source1/fs1/ok_filesystem/*")
# shelltest("zfs mount /test_target1/test_source1/fs1/bad_filesystem")
shelltest("rm /test_source1/fs1/bad_filesystem/*")
shelltest("dd if=/dev/zero of=/dev/zvol/test_source1/fs1/ok_zvol count=1 bs=512k")
def test_verify(self):
self.assertFalse(ZfsAutoverify("test test_target1 --verbose --test".split(" ")).run())
self.assertFalse(ZfsAutoverify("test test_target1 --verbose".split(" ")).run())
#rsync mode
self.assertEqual(1, ZfsAutoverify("test test_target1 --verbose".split(" ")).run())
self.assertEqual(1, ZfsAutoverify("test test_target1 --ssh-source=localhost --verbose --exclude-received".split(" ")).run())
self.assertEqual(1, ZfsAutoverify("test test_target1 --ssh-target=localhost --verbose --exclude-received".split(" ")).run())

33
zfs_autobackup/ZfsAutoverify.py
View File

@ -30,14 +30,19 @@ class ZfsAutoverify(ZfsAuto):
"""extend common parser with extra stuff needed for zfs-autobackup"""
parser=super(ZfsAutoverify, self).get_parser()
group=parser.add_argument_group("Verify options")
group.add_argument('--fs-compare', metavar='METHOD', default="tar", choices=["tar", "rsync"],
help='Compare method to use for filesystems. (tar, rsync) Default: %(default)s ')
return parser
def compare_trees(self , source_node, source_path, target_node, target_path):
"""recursively compare checksums in both trees"""
#NOTE: perhaps support multiple compare methods/commands?
def compare_trees_rsync(self , source_node, source_path, target_node, target_path):
"""recursively compare checksums in both directory trees"""
#currently we use rsync for this.
#NOTE: perhaps support multiple compare implementations?
cmd = ["rsync", "-rcn", "--info=COPY,DEL,MISC,NAME,SYMSAFE", "--msgs2stderr", "--delete" ]
@ -45,44 +50,50 @@ class ZfsAutoverify(ZfsAuto):
if source_node.ssh_to is None and target_node.ssh_to is None:
cmd.append("{}/".format(source_path))
cmd.append("{}/".format(target_path))
source_node.debug("Running rsync locally, on source.")
stdout, stderr = source_node.run(cmd, return_stderr=True)
#source is local
elif source_node.ssh_to is None and target_node.ssh_to is not None:
cmd.append("{}/".format(source_path))
cmd.append("{}:{}/".format(target_node.ssh_to, target_path))
source_node.debug("Running rsync locally, on source.")
stdout, stderr = source_node.run(cmd, return_stderr=True)
#target is local
elif source_node.ssh_to is not None and target_node.ssh_to is None:
cmd.append("{}:{}/".format(source_node.ssh_to, source_path))
cmd.append("{}/".format(target_path))
source_node.debug("Running rsync locally, on target.")
stdout, stderr=target_node.run(cmd, return_stderr=True)
else:
raise Exception("Source and target cant both be remote when using rsync to verify datasets")
raise Exception("Source and target cant both be remote when verifying. (rsync limitation)")
if stderr:
raise Exception("Dataset verify failed, see above list for differences")
def verify_filesystem(self, source_snapshot, source_mnt, target_snapshot, target_mnt):
"""Compare the contents of two zfs filesystem snapshots """
try:
#mount the snapshots
source_snapshot.mount(source_mnt)
target_snapshot.mount(target_mnt)
self.compare_trees(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt)
self.compare_trees_rsync(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt)
finally:
source_snapshot.unmount()
target_snapshot.unmount()
def verify_volume(self, source_dataset, target_dataset):
target_dataset.error("XXX implement me")
pass
def verify_datasets(self, source_node, source_mnt, source_datasets, target_node, target_mnt):
def verify_datasets(self, source_mnt, source_datasets, target_node, target_mnt):
fail_count=0
count = 0
@ -102,6 +113,9 @@ class ZfsAutoverify(ZfsAuto):
source_snapshot = source_dataset.find_common_snapshot(target_dataset)
target_snapshot = target_dataset.find_snapshot(source_snapshot)
if source_snapshot is None or target_snapshot is None:
raise(Exception("Cant find common snapshot"))
target_snapshot.verbose("Verifying...")
if source_dataset.properties['type']=="filesystem":
@ -183,7 +197,6 @@ class ZfsAutoverify(ZfsAuto):
source_mnt, target_mnt=self.create_mountpoints(source_node, target_node)
fail_count = self.verify_datasets(
source_node=source_node,
source_mnt=source_mnt,
source_datasets=source_datasets,
target_mnt=target_mnt,

4
zfs_autobackup/ZfsDataset.py
View File

@ -1102,6 +1102,8 @@ class ZfsDataset:
def mount(self, mount_point):
self.debug("Mounting")
cmd = [
"mount", "-tzfs", self.name, mount_point
]
@ -1110,6 +1112,8 @@ class ZfsDataset:
def unmount(self):
self.debug("Unmounting")
cmd = [
"umount", self.name
]