third-party-mirrors/zfs_autobackup
1
0
Fork 1
mirror of https://github.com/psy0rz/zfs_autobackup.git synced 2025-06-05 01:33:00 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Edwin Eefting 2024-10-05 17:15:36 +02:00
parent 0715a45f11
commit 2f4ea79ff3
No known key found for this signature in database
GPG Key ID: 0F3C35D8E9887737
3 changed files with 80 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
tests/test_zfsautobackup34.py
View File

@ -1,11 +1,12 @@
from basetest import * from basetest import *
class TestZfsAutobackup34(unittest2.TestCase): class TestZfsAutobackup34(unittest2.TestCase):
"""various new 3.4 features""" """various new 3.4 features"""
def setUp(self): def setUp(self):
prepare_zpools() prepare_zpools()
self.longMessage=True self.longMessage = True
def test_no_bookmark_source_support(self): def test_no_bookmark_source_support(self):
"""test if everything is fine when source has no bookmark support (has no features at all even)""" """test if everything is fine when source has no bookmark support (has no features at all even)"""
@ -13,19 +14,19 @@ class TestZfsAutobackup34(unittest2.TestCase):
subprocess.check_call("zpool destroy test_source1", shell=True) subprocess.check_call("zpool destroy test_source1", shell=True)
subprocess.check_call("zpool create -d test_source1 /dev/ram0", shell=True) subprocess.check_call("zpool create -d test_source1 /dev/ram0", shell=True)
shelltest("zpool get all test_source1") shelltest("zpool get all test_source1")
subprocess.check_call("zfs create -p test_source1/fs1/sub", shell=True) # recreate with no features at all subprocess.check_call("zfs create -p test_source1/fs1/sub", shell=True) # recreate with no features at all
subprocess.check_call("zfs set autobackup:test=true test_source1/fs1", shell=True) subprocess.check_call("zfs set autobackup:test=true test_source1/fs1", shell=True)
with mocktime("20101111000001"): with mocktime("20101111000001"):
self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run())
#should fallback on holds on the source snapshot # should fallback on holds on the source snapshot
r=shelltest("zfs holds test_source1/fs1@test-20101111000001") r = shelltest("zfs holds test_source1/fs1@test-20101111000001")
self.assertIn("zfs_autobackup:test", r) self.assertIn("zfs_autobackup:test", r)
def test_no_bookmark_target_support(self): def test_no_bookmark_target_support(self):
"""test if everything is fine when target has no bookmark support (has no features at all even)""" """test if everything is fine when target has no bookmark support (has no features at all even)"""
#NOTE: not sure if its ok if only the source supports bookmarks, so currently zfs-autobackup requires both sides to support bookmarks to enable it. # NOTE: not sure if its ok if only the source supports bookmarks, so currently zfs-autobackup requires both sides to support bookmarks to enable it.
subprocess.check_call("zpool destroy test_target1", shell=True) subprocess.check_call("zpool destroy test_target1", shell=True)
subprocess.check_call("zpool create -d test_target1 /dev/ram2", shell=True) subprocess.check_call("zpool create -d test_target1 /dev/ram2", shell=True)
@ -33,22 +34,21 @@ class TestZfsAutobackup34(unittest2.TestCase):
with mocktime("20101111000001"): with mocktime("20101111000001"):
self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run())
#should fallback on holds on the source snapshot # should fallback on holds on the source snapshot
r=shelltest("zfs holds test_source1/fs1@test-20101111000001") r = shelltest("zfs holds test_source1/fs1@test-20101111000001")
self.assertIn("zfs_autobackup:test", r) self.assertIn("zfs_autobackup:test", r)
def test_select_bookmark_or_snapshot(self): def test_select_bookmark_or_snapshot(self):
"""test if zfs autobackup chooses the most recent common matching dataset when there are both bookmarks and snapshots, some with the wrong GUID""" """test if zfs autobackup chooses the most recent common matching dataset when there are both bookmarks and snapshots, some with the wrong GUID"""
with mocktime("20101111000001"): with mocktime("20101111000001"):
self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run())
#destroy stuff and see if it still selects the correct ones # destroy stuff and see if it still selects the correct ones
shelltest("zfs destroy test_source2/fs2/sub@test-20101111000001") shelltest("zfs destroy test_source2/fs2/sub@test-20101111000001")
shelltest("zfs destroy test_source1/fs1/sub#test-20101111000001") shelltest("zfs destroy test_source1/fs1/sub#test-20101111000001")
#bookmark with incorrect GUID, should fallback to snapshot # bookmark with incorrect GUID, should fallback to snapshot
shelltest("zfs destroy test_source1/fs1#test-20101111000001") shelltest("zfs destroy test_source1/fs1#test-20101111000001")
shelltest("zfs snapshot test_source1/fs1@wrong") shelltest("zfs snapshot test_source1/fs1@wrong")
shelltest("zfs bookmark test_source1/fs1@wrong \#test-20101111000001") shelltest("zfs bookmark test_source1/fs1@wrong \#test-20101111000001")
@ -57,9 +57,8 @@ class TestZfsAutobackup34(unittest2.TestCase):
with mocktime("20101111000002"): with mocktime("20101111000002"):
self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run())
r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS)
r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) self.assertMultiLineEqual(r, """
self.assertMultiLineEqual(r,"""
test_source1 test_source1
test_source1/fs1 test_source1/fs1
test_source1/fs1@test-20101111000001 test_source1/fs1@test-20101111000001
@ -92,20 +91,19 @@ test_target1/test_source2/fs2/sub@test-20101111000001
test_target1/test_source2/fs2/sub@test-20101111000002 test_target1/test_source2/fs2/sub@test-20101111000002
""") """)
#while we're here, check that there are no holds on source common snapshot (since bookmarks replace holds on source side) # while we're here, check that there are no holds on source common snapshot (since bookmarks replace holds on source side)
r=shelltest("zfs holds test_source2/fs2/sub@test-20101111000002") r = shelltest("zfs holds test_source2/fs2/sub@test-20101111000002")
self.assertNotIn("zfs_autobackup:test", r) self.assertNotIn("zfs_autobackup:test", r)
def test_disable_bookmarks(self): def test_disable_bookmarks(self):
"""test if we can disable it on an existing backup with bookmarks, with --no-bookmarks and get the old behaviour (holds on source)""" """test if we can disable it on an existing backup with bookmarks, with --no-bookmarks and get the old behaviour (holds on source)"""
#first with bookmarks enabled # first with bookmarks enabled
with mocktime("20101111000001"): with mocktime("20101111000001"):
self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run())
r=shelltest("zfs list -H -o name -r -t all test_source1") r = shelltest("zfs list -H -o name -r -t all test_source1")
self.assertMultiLineEqual(r,""" self.assertMultiLineEqual(r, """
test_source1 test_source1
test_source1/fs1 test_source1/fs1
test_source1/fs1@test-20101111000001 test_source1/fs1@test-20101111000001
@ -115,12 +113,13 @@ test_source1/fs1/sub@test-20101111000001
test_source1/fs1/sub#test-20101111000001 test_source1/fs1/sub#test-20101111000001
""") """)
#disable it # disable it
with mocktime("20101111000002"): with mocktime("20101111000002"):
self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-bookmarks".split(" ")).run()) self.assertFalse(ZfsAutobackup(
"test test_target1 --no-progress --verbose --allow-empty --no-bookmarks".split(" ")).run())
r=shelltest("zfs list -H -o name -r -t all test_source1") r = shelltest("zfs list -H -o name -r -t all test_source1")
self.assertMultiLineEqual(r,""" self.assertMultiLineEqual(r, """
test_source1 test_source1
test_source1/fs1 test_source1/fs1
test_source1/fs1@test-20101111000001 test_source1/fs1@test-20101111000001
@ -130,12 +129,12 @@ test_source1/fs1/sub@test-20101111000001
test_source1/fs1/sub@test-20101111000002 test_source1/fs1/sub@test-20101111000002
""") """)
#re-enable # re-enable
with mocktime("20101111000003"): with mocktime("20101111000003"):
self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run())
r=shelltest("zfs list -H -o name -r -t all test_source1") r = shelltest("zfs list -H -o name -r -t all test_source1")
self.assertMultiLineEqual(r,""" self.assertMultiLineEqual(r, """
test_source1 test_source1
test_source1/fs1 test_source1/fs1
test_source1/fs1@test-20101111000001 test_source1/fs1@test-20101111000001
@ -150,21 +149,23 @@ test_source1/fs1/sub#test-20101111000003
""") """)
def test_tags(self): def test_tags(self):
with mocktime("20101111000001"): with mocktime("20101111000001"):
self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test1".split(" ")).run()) self.assertFalse(
ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test1".split(" ")).run())
with mocktime("20101111000002"): with mocktime("20101111000002"):
self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test2".split(" ")).run()) self.assertFalse(
ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test2".split(" ")).run())
with mocktime("20101111000003"): with mocktime("20101111000003"):
# make sure the thinner sees and cleans up the old snaphots that have a tag # make sure the thinner sees and cleans up the old snaphots that have a tag
self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --keep-source=2 --keep-target=2".split(" ")).run()) self.assertFalse(ZfsAutobackup(
"test test_target1 --no-progress --verbose --allow-empty --keep-source=2 --keep-target=2".split(
" ")).run())
r=shelltest("zfs list -H -r -t snapshot -o name "+TEST_POOLS) r = shelltest("zfs list -H -r -t snapshot -o name " + TEST_POOLS)
self.assertMultiLineEqual(r,""" self.assertMultiLineEqual(r, """
test_source1/fs1@test-20101111000002_test2 test_source1/fs1@test-20101111000002_test2
test_source1/fs1@test-20101111000003 test_source1/fs1@test-20101111000003
test_source1/fs1/sub@test-20101111000002_test2 test_source1/fs1/sub@test-20101111000002_test2
@ -179,3 +180,35 @@ test_target1/test_source2/fs2/sub@test-20101111000002_test2
test_target1/test_source2/fs2/sub@test-20101111000003 test_target1/test_source2/fs2/sub@test-20101111000003
""") """)
def test_double_send_bookmark(self):
"""test sending the same snaphots to 2 targets, and check if they each use their own bookmark and delete them correctly."""
shelltest("zfs create test_target1/a")
shelltest("zfs create test_target1/b")
# full
with mocktime("20101111000001"):
self.assertFalse(
ZfsAutobackup(
"test test_target1/a --no-progress --verbose --allow-empty --tag tag1".split(" ")).run())
# increment, should be from bookmark
with mocktime("20101111000002"):
self.assertFalse(
ZfsAutobackup(
"test test_target1/a --no-progress --verbose --allow-empty --tag tag1".split(" ")).run())
# to target b, now each has one full + two incrementals, which should be from their own bookmarks.
with mocktime("20101111000003"):
self.assertFalse(
ZfsAutobackup(
"test test_target1/b --no-progress --verbose --no-snapshot --allow-empty".split(
" ")).run())
# result:
# for target a the bookmarks should be at 20101111000002, for target b the bookmarks should be at 20101111000003
r = shelltest("zfs list -H -r -t all -o name " + TEST_POOLS)
self.assertMultiLineEqual(r, """
...
""")

8
zfs_autobackup/ZfsAutobackup.py
View File

@ -357,8 +357,9 @@ class ZfsAutobackup(ZfsAuto):
target_datasets[target_name] = source_dataset target_datasets[target_name] = source_dataset
# NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters:
def sync_datasets(self, source_node, source_datasets, target_node): def sync_datasets(self, source_node, source_datasets, target_node, bookmark_tag):
"""Sync datasets, or thin-only on both sides """Sync datasets, or thin-only on both sides
:type bookmark_tag: str
:type target_node: ZfsNode :type target_node: ZfsNode
:type source_datasets: list of ZfsDataset :type source_datasets: list of ZfsDataset
:type source_node: ZfsNode :type source_node: ZfsNode
@ -419,7 +420,8 @@ class ZfsAutobackup(ZfsAuto):
send_pipes=send_pipes, recv_pipes=recv_pipes, send_pipes=send_pipes, recv_pipes=recv_pipes,
decrypt=self.args.decrypt, encrypt=self.args.encrypt, decrypt=self.args.decrypt, encrypt=self.args.encrypt,
zfs_compressed=self.args.zfs_compressed, force=self.args.force, zfs_compressed=self.args.zfs_compressed, force=self.args.force,
guid_check=not self.args.no_guid_check, use_bookmarks=use_bookmarks) guid_check=not self.args.no_guid_check, use_bookmarks=use_bookmarks,
bookmark_tag=bookmark_tag)
except Exception as e: except Exception as e:
fail_count = fail_count + 1 fail_count = fail_count + 1
@ -556,7 +558,7 @@ class ZfsAutobackup(ZfsAuto):
fail_count = self.sync_datasets( fail_count = self.sync_datasets(
source_node=source_node, source_node=source_node,
source_datasets=source_datasets, source_datasets=source_datasets,
target_node=target_node) target_node=target_node, bookmark_tag=target_dataset.properties['guid'])
# no target specified, run in snapshot-only mode # no target specified, run in snapshot-only mode
else: else:

14
zfs_autobackup/ZfsDataset.py
View File

@ -646,8 +646,9 @@ class ZfsDataset:
return True return True
def bookmark(self): def bookmark(self, tag):
"""Bookmark this snapshot, and return the bookmark""" """Bookmark this snapshot, and return the bookmark."""
# NOTE: we use the tag to add the target_path GUID, so that we can have multiple bookmarks for the same snapshot, but for different target. This is to make sure you can send a backup to two locations, without them interfering with eachothers bookmarks.
if not self.is_snapshot: if not self.is_snapshot:
raise (Exception("Can only bookmark a snapshot!")) raise (Exception("Can only bookmark a snapshot!"))
@ -655,7 +656,7 @@ class ZfsDataset:
self.debug("Bookmarking") self.debug("Bookmarking")
cmd = [ cmd = [
"zfs", "bookmark", self.name, "#" + self.suffix "zfs", "bookmark", self.name, "#" + self.tagless_suffix + self.zfs_node.tag_seperator + tag
] ]
self.zfs_node.run(cmd=cmd) self.zfs_node.run(cmd=cmd)
@ -772,6 +773,7 @@ class ZfsDataset:
cmd.extend(send_pipes) cmd.extend(send_pipes)
self.error(cmd)
output_pipe = self.zfs_node.run(cmd, pipe=True, readonly=True) output_pipe = self.zfs_node.run(cmd, pipe=True, readonly=True)
return output_pipe return output_pipe
@ -1237,7 +1239,7 @@ class ZfsDataset:
def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties,
ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots,
no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check,
use_bookmarks): use_bookmarks, bookmark_tag):
"""sync this dataset's snapshots to target_dataset, while also thinning """sync this dataset's snapshots to target_dataset, while also thinning
out old snapshots along the way. out old snapshots along the way.
@ -1256,6 +1258,8 @@ class ZfsDataset:
:type also_other_snapshots: bool :type also_other_snapshots: bool
:type no_send: bool :type no_send: bool
:type guid_check: bool :type guid_check: bool
:type use_bookmarks: bool
:type bookmark_tag: str
""" """
# self.verbose("-> {}".format(target_dataset)) # self.verbose("-> {}".format(target_dataset))
@ -1344,7 +1348,7 @@ class ZfsDataset:
# bookmark common snapshot on source, or use holds if bookmarks are not enabled. # bookmark common snapshot on source, or use holds if bookmarks are not enabled.
if use_bookmarks: if use_bookmarks:
source_bookmark = source_snapshot.bookmark() source_bookmark = source_snapshot.bookmark(bookmark_tag)
# note: destroy source_snapshot when obsolete at this point? # note: destroy source_snapshot when obsolete at this point?
else: else:
source_bookmark = None source_bookmark = None