From 7b8b536d53d775fe50633e82737e687b1e844629 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 14 Jul 2020 20:12:11 +0200 Subject: [PATCH] update docs for upcoming change. more tests --- README.md | 21 +++++++++- test_zfsautobackup.py | 89 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 715e44d..3e950d9 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Since its using ZFS commands, you can see what its actually doing by specifying An important feature thats missing from other tools is a reliable `--test` option: This allows you to see what zfs-autobackup will do and tune your parameters. It will do everything, except make changes to your zfs datasets. -Another nice thing is progress reporting with `--progress`. Its very useful with HUGE datasets, when you want to know how many hours/days it will take. +Another nice thing is progress reporting: Its very useful with HUGE datasets, when you want to know how many hours/days it will take. zfs-autobackup tries to be the easiest to use backup tool for zfs. @@ -61,6 +61,7 @@ zfs-autobackup tries to be the easiest to use backup tool for zfs. * Easy to debug and has a test-mode. Actual unix commands are printed. * Uses **progressive thinning** for older snapshots. * Uses zfs-holds on important snapshots so they cant be accidentally destroyed. +* Automatic resuming of failed transfers. * Easy installation: * Just install zfs-autobackup via pip, or download it manually. * Written in python and uses zfs-commands, no 3rd party dependency's or libraries. @@ -296,6 +297,24 @@ The thinner operates "stateless": There is nothing in the name or properties of Note that the thinner will ONLY destroy snapshots that are matching the naming pattern of zfs-autobackup. If you use `--other-snapshots`, it wont destroy those snapshots after replicating them to the target. +### Destroying missing datasets + +When a dataset has been destroyed on the source, but still exists on the target we call it a missing dataset. Missing datasets will be still thinned out according to the schedule. + +The final snapshot will never be destroyed, unless you specify a **deadline** with the `--destroy-missing` option: + +In that case it will look at the last snapshot we took and determine if is older than the deadline you specified. e.g: `--destroy-missing 30d` will start destroying things 30 days after the last snapshot. + +#### After the deadline + +When the deadline is passed, all our snapshots, except the last one will be destroyed. Irregardless of the normal thinning schedule. + +The dataset has to have the following properties to be finally really destroyed: + +* The dataset has no direct child-filesystems or volumes. +* The only snapshot left is the last one created by zfs-autobackup. +* The remaining snapshot has no clones. + ### Thinning schedule The default thinning schedule is: `10,1d1w,1w1m,1m1y`. diff --git a/test_zfsautobackup.py b/test_zfsautobackup.py index 8fd866c..794e942 100644 --- a/test_zfsautobackup.py +++ b/test_zfsautobackup.py @@ -14,6 +14,7 @@ class TestZfsAutobackup(unittest2.TestCase): self.assertEqual(ZfsAutobackup("test test_target1 --keep-source -1".split(" ")).run(), 255) def test_snapshotmode(self): + """test snapshot tool mode""" with patch('time.strftime', return_value="20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose".split(" ")).run()) @@ -24,9 +25,9 @@ class TestZfsAutobackup(unittest2.TestCase): with patch('time.strftime', return_value="20101111000002"): self.assertFalse(ZfsAutobackup("test --verbose --allow-empty --keep-source 0".split(" ")).run()) - #on source: only has 1 and 2 + #on source: only has 1 and 2 (1 was hold) #on target: has 0 and 1 - + #XXX: r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) self.assertMultiLineEqual(r,""" test_source1 @@ -63,8 +64,6 @@ test_target1/test_source2/fs2/sub@test-20101111000001 def test_defaults(self): with self.subTest("no datasets selected"): - #should resume and succeed - with OutputIO() as buf: with redirect_stderr(buf): with patch('time.strftime', return_value="20101111000000"): @@ -175,8 +174,50 @@ test_target1/test_source2/fs2/sub@test-20101111000000 userrefs 0 - test_target1/test_source2/fs2/sub@test-20101111000001 userrefs 1 - """) + #make sure time handling is correctly. try to make snapshots a year appart and verify that only snapshots mostly 1y old are kept + with self.subTest("test time checking"): + with patch('time.strftime', return_value="20111111000000"): + self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --verbose".split(" ")).run()) + time_str="20111112000000" #month in the "future" + future_timestamp=time_secs=time.mktime(time.strptime(time_str,"%Y%m%d%H%M%S")) + with patch('time.time', return_value=future_timestamp): + with patch('time.strftime', return_value="20111111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --verbose --keep-source 1y1y --keep-target 1d1y".split(" ")).run()) + + + r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) + self.assertMultiLineEqual(r,""" +test_source1 +test_source1/fs1 +test_source1/fs1@test-20111111000000 +test_source1/fs1@test-20111111000001 +test_source1/fs1/sub +test_source1/fs1/sub@test-20111111000000 +test_source1/fs1/sub@test-20111111000001 +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20111111000000 +test_source2/fs2/sub@test-20111111000001 +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@test-20111111000000 +test_target1/test_source1/fs1@test-20111111000001 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20111111000000 +test_target1/test_source1/fs1/sub@test-20111111000001 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20111111000000 +test_target1/test_source2/fs2/sub@test-20111111000001 +""") + def test_ignore_othersnaphots(self): @@ -757,6 +798,44 @@ test_target1/test_source2/fs2/sub@test-20101111000001 """) + def test_migrate(self): + """test migration from other snapshotting systems. zfs-autobackup should be able to continue from any common snapshot, not just its own.""" + + shelltest("zfs snapshot test_source1/fs1@migrate1") + shelltest("zfs create test_target1/test_source1") + shelltest("zfs send test_source1/fs1@migrate1| zfs recv test_target1/test_source1/fs1") + + with patch('time.strftime', return_value="20101111000000"): + self.assertFalse(ZfsAutobackup("test test_target1 --verbose".split(" ")).run()) + + r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) + self.assertMultiLineEqual(r,""" +test_source1 +test_source1/fs1 +test_source1/fs1@migrate1 +test_source1/fs1@test-20101111000000 +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000000 +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000000 +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@migrate1 +test_target1/test_source1/fs1@test-20101111000000 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000000 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000000 +""") + + ########################### # TODO: @@ -764,3 +843,5 @@ test_target1/test_source2/fs2/sub@test-20101111000001 self.skipTest("todo: later when travis supports zfs 0.8") + +