diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 8c6672e..50fb169 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -1,7 +1,7 @@ name: Regression tests -on: ["push", "pull_request"] +on: [ "push", "pull_request" ] @@ -25,3 +25,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: coveralls --service=github || true + diff --git a/.gitignore b/.gitignore index 723d68a..df2d0e8 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ python2.env venv .idea password.sh +.DS_Store diff --git a/README.md b/README.md index 86eae2d..0445109 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ - # ZFS autobackup [![Tests](https://github.com/psy0rz/zfs_autobackup/workflows/Regression%20tests/badge.svg)](https://github.com/psy0rz/zfs_autobackup/actions?query=workflow%3A%22Regression+tests%22) [![Coverage Status](https://coveralls.io/repos/github/psy0rz/zfs_autobackup/badge.svg)](https://coveralls.io/github/psy0rz/zfs_autobackup) [![Python Package](https://github.com/psy0rz/zfs_autobackup/workflows/Upload%20Python%20Package/badge.svg)](https://pypi.org/project/zfs-autobackup/) @@ -10,26 +9,31 @@ ZFS-autobackup tries to be the most reliable and easiest to use tool, while havi You can either use it as a **backup** tool, **replication** tool or **snapshot** tool. -You can select what to backup by setting a custom `ZFS property`. This makes it easy to add/remove specific datasets, or just backup your whole pool. +You can select what to backup by setting a custom `ZFS property`. This makes it easy to add/remove specific datasets, or +just backup your whole pool. -Other settings are just specified on the commandline: Simply setup and test your zfs-autobackup command and fix all the issues you might encounter. When you're done you can just copy/paste your command to a cron or script. +Other settings are just specified on the commandline: Simply setup and test your zfs-autobackup command and fix all the +issues you might encounter. When you're done you can just copy/paste your command to a cron or script. -Since it's using ZFS commands, you can see what it's actually doing by specifying `--debug`. This also helps a lot if you run into some strange problem or errors. You can just copy-paste the command that fails and play around with it on the commandline. (something I missed in other tools) +Since it's using ZFS commands, you can see what it's actually doing by specifying `--debug`. This also helps a lot if +you run into some strange problem or errors. You can just copy-paste the command that fails and play around with it on +the commandline. (something I missed in other tools) -An important feature that's missing from other tools is a reliable `--test` option: This allows you to see what zfs-autobackup will do and tune your parameters. It will do everything, except make changes to your system. +An important feature that's missing from other tools is a reliable `--test` option: This allows you to see what +zfs-autobackup will do and tune your parameters. It will do everything, except make changes to your system. ## Features * Works across operating systems: Tested with **Linux**, **FreeBSD/FreeNAS** and **SmartOS**. -* Low learning curve: no complex daemons or services, no additional software or networking needed. +* Low learning curve: no complex daemons or services, no additional software or networking needed. * Plays nicely with existing replication systems. (Like Proxmox HA) -* Automatically selects filesystems to backup by looking at a simple ZFS property. +* Automatically selects filesystems to backup by looking at a simple ZFS property. * Creates consistent snapshots. (takes all snapshots at once, atomicly.) * Multiple backups modes: - * Backup local data on the same server. - * "push" local data to a backup-server via SSH. - * "pull" remote data from a server via SSH and backup it locally. - * "pull+push": Zero trust between source and target. + * Backup local data on the same server. + * "push" local data to a backup-server via SSH. + * "pull" remote data from a server via SSH and backup it locally. + * "pull+push": Zero trust between source and target. * Can be scheduled via simple cronjob or run directly from commandline. * Also supports complex backup geometries. * ZFS encryption support: Can decrypt / encrypt or even re-encrypt datasets during transfer. @@ -48,14 +52,11 @@ An important feature that's missing from other tools is a reliable `--test` opti * Complete and clean logging. * All code is regression tested against actual ZFS environments. * Easy installation: - * Just install zfs-autobackup via pip. - * Only needs to be installed on one side. - * Written in python and uses zfs-commands, no special 3rd party dependency's or compiled libraries needed. - * No annoying config files or properties. -* Easy to use in scripts, for example https://github.com/psy0rz/zfs_autobackup/wiki/Example-Proxmox + * Just install zfs-autobackup via pip. + * Only needs to be installed on one side. + * Written in python and uses zfs-commands, no special 3rd party dependency's or compiled libraries needed. + * No annoying config files or properties. - - ## Getting started Please look at our wiki to [Get started](https://github.com/psy0rz/zfs_autobackup/wiki). @@ -66,6 +67,6 @@ Or read the [Full manual](https://github.com/psy0rz/zfs_autobackup/wiki/Manual) This project was sponsored by: -* JetBrains +* JetBrains * https://rsync.net * [DatuX](https://www.datux.nl) diff --git a/requirements.txt b/requirements.txt index 2319de0..b6bc9a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ colorama -argparse coverage python-coveralls unittest2 diff --git a/setup.py b/setup.py index 26201ec..8547daa 100644 --- a/setup.py +++ b/setup.py @@ -26,12 +26,11 @@ setuptools.setup( packages=setuptools.find_packages(), classifiers=[ - "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", ], - python_requires='>=2.7', + python_requires='>=3.2', install_requires=[ "colorama", "argparse" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..66b16a7 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,6 @@ +import os +import sys + +_here = os.path.dirname(os.path.abspath(__file__)) +if _here not in sys.path: + sys.path.insert(0, _here) diff --git a/tests/autoremote b/tests/autoremote new file mode 100755 index 0000000..c41e7e9 --- /dev/null +++ b/tests/autoremote @@ -0,0 +1,4 @@ +#!/bin/bash +# to run one: -k test_incremental_resume + +find tests/*.py zfs_autobackup/*.py | entr -r ./tests/remoteruntest $@ diff --git a/tests/remoteruntest b/tests/remoteruntest new file mode 100755 index 0000000..f24208e --- /dev/null +++ b/tests/remoteruntest @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +HOST=root@ws2 +DST=/home/psy/zfs_autobackup + +rsync --delete -avx . "$HOST:$DST" + +ssh $HOST "cd $DST && ./tests/run_test $@" diff --git a/tests/run_test b/tests/run_test index 1824a5e..c400fd1 100755 --- a/tests/run_test +++ b/tests/run_test @@ -1,5 +1,6 @@ #!/bin/bash -#run one test. start from main directory - -python -m unittest discover tests $@ -vvvf +#run all or one test. start from main directory +# to run one: -k test_incremental_resume +cd tests +python -m unittest discover .. $@ -vvvf diff --git a/tests/run_tests b/tests/run_tests index 3b5d6ec..ea7f6c0 100755 --- a/tests/run_tests +++ b/tests/run_tests @@ -31,7 +31,8 @@ modprobe brd rd_size=512000 umount /tmp/ZfsCheck* -coverage run --branch --source zfs_autobackup -m unittest discover -vvvvf $SCRIPTDIR $@ 2>&1 +cd $SCRIPTDIR +coverage run --branch --source zfs_autobackup -m unittest discover -vvvvf .. $@ 2>&1 EXIT=$? echo diff --git a/tests/test_blockhasher.py b/tests/test_blockhasher.py index 957d487..3214d6b 100644 --- a/tests/test_blockhasher.py +++ b/tests/test_blockhasher.py @@ -20,28 +20,28 @@ class TestBlockHasher(unittest2.TestCase): def test_empty(self): block_hasher = BlockHasher(count=1) self.assertEqual( - list(block_hasher.generate("tests/data/empty")), + list(block_hasher.generate("data/empty")), [] ) def test_partial(self): block_hasher = BlockHasher(count=1) self.assertEqual( - list(block_hasher.generate("tests/data/partial")), + list(block_hasher.generate("data/partial")), [(0, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1")] ) def test_whole(self): block_hasher = BlockHasher(count=1) self.assertEqual( - list(block_hasher.generate("tests/data/whole")), + list(block_hasher.generate("data/whole")), [(0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7")] ) def test_whole2(self): block_hasher = BlockHasher(count=1) self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2")), + list(block_hasher.generate("data/whole_whole2")), [ (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798") @@ -51,7 +51,7 @@ class TestBlockHasher(unittest2.TestCase): def test_wwp(self): block_hasher = BlockHasher(count=1) self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), # whole (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), # whole2 @@ -62,7 +62,7 @@ class TestBlockHasher(unittest2.TestCase): def test_wwp_count2(self): block_hasher = BlockHasher(count=2) self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ (0, "959e6b58078f0cfd2fb3d37e978fda51820473ff"), # whole_whole2 (1, "642027d63bb0afd7e0ba197f2c66ad03e3d70de1") # partial @@ -72,7 +72,7 @@ class TestBlockHasher(unittest2.TestCase): def test_big(self): block_hasher = BlockHasher(count=10) self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ (0, "309ffffba2e1977d12f3b7469971f30d28b94bd8"), # whole_whole2_partial ]) @@ -80,32 +80,32 @@ class TestBlockHasher(unittest2.TestCase): def test_blockhash_compare(self): #no errors block_hasher = BlockHasher(count=1) - generator = block_hasher.generate("tests/data/whole_whole2_partial") - self.assertEqual([], list(block_hasher.compare("tests/data/whole_whole2_partial", generator))) + generator = block_hasher.generate("data/whole_whole2_partial") + self.assertEqual([], list(block_hasher.compare("data/whole_whole2_partial", generator))) #compare file is smaller (EOF errors) block_hasher = BlockHasher(count=1) - generator = block_hasher.generate("tests/data/whole_whole2_partial") + generator = block_hasher.generate("data/whole_whole2_partial") self.assertEqual( [(1, '2e863f1fcccd6642e4e28453eba10d2d3f74d798', 'EOF'), (2, '642027d63bb0afd7e0ba197f2c66ad03e3d70de1', 'EOF')], - list(block_hasher.compare("tests/data/whole", generator))) + list(block_hasher.compare("data/whole", generator))) #no errors, huge chunks block_hasher = BlockHasher(count=10) - generator = block_hasher.generate("tests/data/whole_whole2_partial") - self.assertEqual([], list(block_hasher.compare("tests/data/whole_whole2_partial", generator))) + generator = block_hasher.generate("data/whole_whole2_partial") + self.assertEqual([], list(block_hasher.compare("data/whole_whole2_partial", generator))) # different order to make sure seek functions are ok block_hasher = BlockHasher(count=1) - checksums = list(block_hasher.generate("tests/data/whole_whole2_partial")) + checksums = list(block_hasher.generate("data/whole_whole2_partial")) checksums.reverse() - self.assertEqual([], list(block_hasher.compare("tests/data/whole_whole2_partial", checksums))) + self.assertEqual([], list(block_hasher.compare("data/whole_whole2_partial", checksums))) def test_skip1(self): block_hasher = BlockHasher(count=1, skip=1) self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), # whole # (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), # whole2 @@ -115,7 +115,7 @@ class TestBlockHasher(unittest2.TestCase): #should continue the pattern on the next file: self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ # (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), # whole (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), # whole2 @@ -126,7 +126,7 @@ class TestBlockHasher(unittest2.TestCase): def test_skip6(self): block_hasher = BlockHasher(count=1, skip=6) self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), # whole # (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), # whole2 @@ -136,7 +136,7 @@ class TestBlockHasher(unittest2.TestCase): #all blocks of next file are skipped self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ # (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), # whole # (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), # whole2 @@ -146,7 +146,7 @@ class TestBlockHasher(unittest2.TestCase): #first block of this one is the 6th to be skipped: self.assertEqual( - list(block_hasher.generate("tests/data/whole_whole2_partial")), + list(block_hasher.generate("data/whole_whole2_partial")), [ # (0, "3c0bf91170d873b8e327d3bafb6bc074580d11b7"), # whole (1, "2e863f1fcccd6642e4e28453eba10d2d3f74d798"), # whole2 diff --git a/tests/test_clone.py b/tests/test_clone.py new file mode 100644 index 0000000..7ee94ea --- /dev/null +++ b/tests/test_clone.py @@ -0,0 +1,300 @@ +from basetest import * + + +class TestZfsAutobackupClone(unittest2.TestCase): + """ZFS clone replication: source datasets that are themselves clones get + replicated to the target as clones of the corresponding target-side origin + snapshot, via 'zfs send -i @'.""" + + def setUp(self): + prepare_zpools() + self.longMessage = True + + def test_clone_happy_path(self): + """Single run, both origin and clone selected. The origin snapshot is + auto-included because a clone in the selection depends on it, so the user + doesn't need --other-snapshots.""" + + shelltest("zfs snapshot test_source1/fs1@base") + shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") + shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertNotIn("guid", combined) # testing can give guid errors + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose".split(" ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@base +test_source1/fs1@test-20101111000000 +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000000 +test_source1/fs1_clone +test_source1/fs1_clone@test-20101111000000 +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000000 +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@base +test_target1/test_source1/fs1@test-20101111000000 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000000 +test_target1/test_source1/fs1_clone +test_target1/test_source1/fs1_clone@test-20101111000000 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000000 +""") + + origin = shelltest("zfs get -H -o value origin test_target1/test_source1/fs1_clone").strip() + self.assertEqual(origin, "test_target1/test_source1/fs1@base") + + def test_clone_origin_not_on_target_falls_back_to_full_send(self): + """If the clone's origin isn't replicated, warn and fall back to full send.""" + + shelltest("zfs inherit autobackup:test test_source1/fs1") + shelltest("zfs snapshot test_source1/fs1@base") + shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") + shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertIn("Cannot replicate as clone", combined) + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertIn("Cannot replicate as clone", combined) + self.assertNotIn("STDERR", combined) + + origin = shelltest("zfs get -H -o value origin test_target1/test_source1/fs1_clone").strip() + self.assertEqual(origin, "-") + + def test_clone_after_initial_origin_backup(self): + """First run backs up the origin; second run replicates a clone of it.""" + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose".split(" ")).run()) + + shelltest("zfs clone test_source1/fs1@test-20101111000000 test_source1/fs1_clone") + shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + origin = shelltest("zfs get -H -o value origin test_target1/test_source1/fs1_clone").strip() + self.assertEqual(origin, "test_target1/test_source1/fs1@test-20101111000000") + + def test_no_clone_flag_disables_clone_replication(self): + """--no-clone reverts to the previous full-send behaviour for clones.""" + + shelltest("zfs snapshot test_source1/fs1@base") + shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") + shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-clone --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-clone".split(" ")).run()) + + origin = shelltest("zfs get -H -o value origin test_target1/test_source1/fs1_clone").strip() + self.assertEqual(origin, "-") + + def test_subsequent_incremental_after_clone_send(self): + """A second run after a clone has been sent is a plain incremental.""" + + shelltest("zfs snapshot test_source1/fs1@base") + shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") + shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose".split(" ")).run()) + + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + r = shelltest("zfs list -H -o name -t snapshot test_target1/test_source1/fs1_clone") + self.assertMultiLineEqual(r, """ +test_target1/test_source1/fs1_clone@test-20101111000000 +test_target1/test_source1/fs1_clone@test-20101111000001 +""") + + origin = shelltest("zfs get -H -o value origin test_target1/test_source1/fs1_clone").strip() + self.assertEqual(origin, "test_target1/test_source1/fs1@base") + + def test_topological_sort_processes_origin_before_clone(self): + """When the clone alphabetically precedes its origin, the topo sort still + processes the origin first so the clone can pin to its target origin.""" + + shelltest("zfs snapshot test_source1/fs1@base") + shelltest("zfs clone test_source1/fs1@base test_source1/aaa_clone") + shelltest("zfs set autobackup:test=true test_source1/aaa_clone") + + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose".split(" ")).run()) + + origin = shelltest("zfs get -H -o value origin test_target1/test_source1/aaa_clone").strip() + self.assertEqual(origin, "test_target1/test_source1/fs1@base") + + def test_test_mode_makes_no_changes_for_clones(self): + """--test must not modify anything, even when clone handling is exercised.""" + + shelltest("zfs snapshot test_source1/fs1@base") + shelltest("zfs clone test_source1/fs1@base test_source1/fs1_clone") + shelltest("zfs set autobackup:test=true test_source1/fs1_clone") + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertNotIn("STDERR", combined) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") + self.assertMultiLineEqual(r, """ +test_target1 +""") + + def test_promoted_clone_origin_on_namespace_descendant_warns_and_falls_back(self): + """After 'zfs promote', the original parent becomes a clone whose origin lives + on one of its own namespace descendants. zfs recv can't land a clone-creating + stream on top of the placeholder we have to create for that descendant, so we + detect this upfront, warn, and fall back to a full send (which will itself + fail at the placeholder — user can destroy target and rerun, or promote the + source dataset to flip the lineage back).""" + + shelltest("zfs snapshot test_source1/fs1@s1") + shelltest("zfs clone test_source1/fs1@s1 test_source1/fs1/inner_clone") + shelltest("zfs promote test_source1/fs1/inner_clone") + + origin = shelltest("zfs get -H -o value origin test_source1/fs1").strip() + self.assertEqual(origin, "test_source1/fs1/inner_clone@s1") + + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + result = ZfsAutobackup( + "test test_target1 --no-progress --verbose --test -F".split(" ")).run() + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertFalse(result) + self.assertIn("lives on a namespace descendant", combined) + self.assertNotIn("STDERR", combined) + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000000"): + result = ZfsAutobackup( + "test test_target1 --no-progress --verbose -F".split(" ")).run() + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertFalse(result) + self.assertIn("lives on a namespace descendant", combined) diff --git a/tests/test_destroymissing.py b/tests/test_destroymissing.py index 89eb4ec..8e9edb2 100644 --- a/tests/test_destroymissing.py +++ b/tests/test_destroymissing.py @@ -1,4 +1,3 @@ - from basetest import * @@ -6,125 +5,127 @@ class TestZfsNode(unittest2.TestCase): def setUp(self): prepare_zpools() - self.longMessage=True + self.longMessage = True - - - def test_destroymissing(self): - - #initial backup - with mocktime("19101111000000"): #1000 years in past + def test_destroymissing(self): + # initial backup + with mocktime("19101111000000"): # 1000 years in past self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds".split(" ")).run()) - with mocktime("20101111000000"): #far in past - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) - + with mocktime("20101111000000"): # far in past + self.assertFalse( + ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) with self.subTest("Should do nothing yet"): with OutputIO() as buf: with redirect_stdout(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split( + " ")).run()) print(buf.getvalue()) self.assertNotIn(": Destroy missing", buf.getvalue()) - with self.subTest("missing dataset of us that still has children"): - - #just deselect it so it counts as 'missing' + # just deselect it so it counts as 'missing' shelltest("zfs set autobackup:test=child test_source1/fs1") with OutputIO() as buf: with redirect_stdout(buf), redirect_stderr(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split( + " ")).run()) print(buf.getvalue()) - #should have done the snapshot cleanup for destoy missing: + # should have done the snapshot cleanup for destoy missing: self.assertIn("fs1@test-19101111000000: Destroying", buf.getvalue()) self.assertIn("fs1: Destroy missing: Still has children here.", buf.getvalue()) shelltest("zfs inherit autobackup:test test_source1/fs1") - with self.subTest("Normal destroyed leaf"): shelltest("zfs destroy -r test_source1/fs1/sub") - #wait for deadline of last snapshot + # wait for deadline of last snapshot with OutputIO() as buf: with redirect_stdout(buf): - #100y: lastest should not be old enough, while second to latest snapshot IS old enough: - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 100y".split(" ")).run()) + # 100y: lastest should not be old enough, while second to latest snapshot IS old enough: + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 100y".split( + " ")).run()) print(buf.getvalue()) self.assertIn(": Waiting for deadline", buf.getvalue()) - #past deadline, destroy + # past deadline, destroy with OutputIO() as buf: with redirect_stdout(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 1y".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 1y".split( + " ")).run()) print(buf.getvalue()) self.assertIn("sub: Destroying", buf.getvalue()) - with self.subTest("Leaf with other snapshot still using it"): shelltest("zfs destroy -r test_source1/fs1") shelltest("zfs snapshot -r test_target1/test_source1/fs1@other1") - with OutputIO() as buf: with redirect_stdout(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split( + " ")).run()) print(buf.getvalue()) - #cant finish because still in use: + # cant finish because still in use: self.assertIn("fs1: Destroy missing: Still in use", buf.getvalue()) shelltest("zfs destroy test_target1/test_source1/fs1@other1") - with self.subTest("In use by clone"): shelltest("zfs clone test_target1/test_source1/fs1@test-20101111000000 test_target1/clone1") with OutputIO() as buf: with redirect_stdout(buf), redirect_stderr(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split( + " ")).run()) print(buf.getvalue()) - #now tries to destroy our own last snapshot (before the final destroy of the dataset) + # now tries to destroy our own last snapshot (before the final destroy of the dataset) self.assertIn("fs1@test-20101111000000: Destroying", buf.getvalue()) - #but cant finish because still in use: + # but cant finish because still in use: self.assertIn("fs1: Error during --destroy-missing", buf.getvalue()) shelltest("zfs destroy test_target1/clone1") - with self.subTest("Should leave test_source1 parent"): - with OutputIO() as buf: with redirect_stdout(buf), redirect_stderr(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split( + " ")).run()) print(buf.getvalue()) - #should have done the snapshot cleanup for destoy missing: + # should have done the snapshot cleanup for destoy missing: self.assertIn("fs1: Destroying", buf.getvalue()) with OutputIO() as buf: with redirect_stdout(buf), redirect_stderr(buf): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-missing 0s".split( + " ")).run()) print(buf.getvalue()) - #on second run it sees the dangling ex-parent but doesnt know what to do with it (since it has no own snapshot) + # on second run it sees the dangling ex-parent but doesnt know what to do with it (since it has no own snapshot) self.assertIn("test_source1: Destroy missing: has no snapshots made by us", buf.getvalue()) - - - - #end result - r=shelltest("zfs list -H -o name -r -t all test_target1") - self.assertMultiLineEqual(r,""" + # end result + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") + self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 test_target1/test_source2 diff --git a/tests/test_encryption.py b/tests/test_encryption.py index 2f08ae3..1a5ecbc 100644 --- a/tests/test_encryption.py +++ b/tests/test_encryption.py @@ -2,6 +2,7 @@ from zfs_autobackup.CmdPipe import CmdPipe from basetest import * import time + # We have to do a LOT to properly test encryption/decryption/raw transfers # # For every scenario we need at least: @@ -20,7 +21,6 @@ import time class TestZfsEncryption(unittest2.TestCase): - def setUp(self): prepare_zpools() @@ -40,7 +40,9 @@ class TestZfsEncryption(unittest2.TestCase): # create encrypted source dataset shelltest("rm /tmp/zfstest.key 2>/dev/null;true") shelltest("echo {} > /tmp/zfstest.key".format(key)) - shelltest("zfs create -o keylocation=file:///tmp/zfstest.key -o keyformat=passphrase -o encryption=on {}".format(path)) + shelltest( + "zfs create -o keylocation=file:///tmp/zfstest.key -o keyformat=passphrase -o encryption=on {}".format( + path)) if unload_key: shelltest("zfs unmount {}".format(path)) @@ -48,23 +50,30 @@ class TestZfsEncryption(unittest2.TestCase): # r=shelltest("dd if=/dev/zero of=/test_source1/fs1/enc1/data.txt bs=200000 count=1") - def test_raw(self): + def test_raw(self): """send encrypted data unaltered (standard operation)""" self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") - self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsourcekeyless", unload_key=True) # raw mode shouldn't need a key + self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsourcekeyless", + unload_key=True) # raw mode shouldn't need a key self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --no-snapshot".split( + " ")).run()) with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --no-snapshot".split( + " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") - self.assertMultiLineEqual(r,""" + self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_target1 encryptionroot - - test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget - @@ -86,19 +95,28 @@ test_target1/test_source2/fs2 encryption test_target1/test_source2/fs2/sub encryptionroot - - """) - def test_decrypt(self): + def test_decrypt(self): """decrypt data and store unencrypted (--decrypt)""" self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --decrypt --allow-empty".split( + " ")).run()) + # NOTE: this also tests if sending a source to a second target works correctly (this failed during bookmark implementation) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --debug".split( + " ")).run()) with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --decrypt --allow-empty".split( + " ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot".split( + " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") self.assertEqual(r, """ @@ -121,19 +139,27 @@ test_target1/test_source2/fs2 encryptionroot - test_target1/test_source2/fs2/sub encryptionroot - - """) - def test_encrypt(self): + def test_encrypt(self): """send normal data set and store encrypted on the other side (--encrypt) issue #60 """ self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --clear-mountpoint".split( + " ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --clear-mountpoint".split( + " ")).run()) with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --clear-mountpoint".split( + " ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --clear-mountpoint".split( + " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") self.assertEqual(r, """ @@ -164,16 +190,18 @@ test_target1/test_source2/fs2/sub encryptionroot - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) + "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --clear-mountpoint".split( + " ")).run()) self.assertFalse(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --clear-mountpoint".split( " ")).run()) with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received".split(" ")).run()) + "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty".split( + " ")).run()) self.assertFalse(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --exclude-received".split( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot".split( " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") @@ -197,10 +225,7 @@ test_target1/test_source2/fs2 encryptionroot - test_target1/test_source2/fs2/sub encryptionroot - - """) - - - - def test_raw_invalid_snapshot(self): + def test_raw_invalid_snapshot(self): """in raw mode, its not allowed to have any newer snaphots on target, #219""" self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") @@ -208,18 +233,19 @@ test_target1/test_source2/fs2/sub encryptionroot - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress".split(" ")).run()) - #this is invalid in raw mode + # this is invalid in raw mode shelltest("zfs snapshot test_target1/test_source1/fs1/encryptedsource@incompatible") with mocktime("20101111000001"): - #should fail because of incompatble snapshot - self.assertEqual(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty".split(" ")).run(),1) - #should destroy incompatible and continue - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --no-snapshot --destroy-incompatible".split(" ")).run()) - + # should fail because of incompatble snapshot + self.assertEqual(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty".split(" ")).run(), + 1) + # should destroy incompatible and continue + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --no-snapshot --destroy-incompatible".split(" ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") - self.assertMultiLineEqual(r,""" + self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_target1 encryptionroot - - test_target1/test_source1 encryptionroot - - @@ -231,16 +257,16 @@ test_target1/test_source2/fs2 encryptionroot - test_target1/test_source2/fs2/sub encryptionroot - - """) - - def test_resume_encrypt_with_no_key(self): + def test_resume_encrypt_with_no_key(self): """test what happens if target encryption key not loaded (this led to a kernel crash of freebsd with 2.1.x i think) while trying to resume""" self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") - with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --clear-mountpoint".split( + " ")).run()) r = shelltest("zfs set compress=off test_source1 test_target1") @@ -253,7 +279,7 @@ test_target1/test_source2/fs2/sub encryptionroot - # should fail and leave resume token with mocktime("20101111000001"): self.assertTrue(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --clear-mountpoint".split( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --clear-mountpoint".split( " ")).run()) # # free up space @@ -270,44 +296,39 @@ test_target1/test_source2/fs2/sub encryptionroot - # resume should fail with mocktime("20101111000001"): self.assertEqual(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint".split( - " ")).run(),3) + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --no-snapshot --clear-mountpoint".split( + " ")).run(), 3) + # NOTE: On some versions this leaves 2 weird sub-datasets that should'nt be there (its probably a zfs bug?) + # so we ignore this, and just make sure the backup resumes correctly after reloading the key. + # r = shelltest("zfs get -r -t snapshot,filesystem encryptionroot test_target1") + # self.assertEqual(r, """ + # NAME PROPERTY VALUE SOURCE + # test_target1 encryptionroot - - + # test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1@test-20101111000000 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - + # test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000000 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - + # test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000001 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - + # test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1/sub/sub encryptionroot - - + # test_target1/encryptedtarget/test_source1/fs1/sub/sub@test-20101111000001 encryptionroot - - + # test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source2/fs2/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source2/fs2/sub/sub encryptionroot - - + # test_target1/encryptedtarget/test_source2/fs2/sub/sub@test-20101111000001 encryptionroot - - + # """) - -#NOTE: On some versions this leaves 2 weird sub-datasets that should'nt be there (its probably a zfs bug?) -#so we ignore this, and just make sure the backup resumes correctly after reloading the key. -# r = shelltest("zfs get -r -t all encryptionroot test_target1") -# self.assertEqual(r, """ -# NAME PROPERTY VALUE SOURCE -# test_target1 encryptionroot - - -# test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1@test-20101111000000 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - -# test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000000 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - -# test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000001 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - -# test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1/sub/sub encryptionroot - - -# test_target1/encryptedtarget/test_source1/fs1/sub/sub@test-20101111000001 encryptionroot - - -# test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source2/fs2/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source2/fs2/sub/sub encryptionroot - - -# test_target1/encryptedtarget/test_source2/fs2/sub/sub@test-20101111000001 encryptionroot - - -# """) - - - - #reload key and resume correctly. + # reload key and resume correctly. self.load_key("22222222", "test_target1/encryptedtarget") # resume should complete with mocktime("20101111000001"): self.assertEqual(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint".split( - " ")).run(),0) - + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --no-snapshot --clear-mountpoint --debug".split( + " ")).run(), 0) diff --git a/tests/test_executenode.py b/tests/test_executenode.py index 1b1db1f..cee137c 100644 --- a/tests/test_executenode.py +++ b/tests/test_executenode.py @@ -202,6 +202,25 @@ class TestExecuteNode(unittest2.TestCase): self.assertEqual(result, ["tEst"]) + def test_script_remote_ssh_config_flat_argv(self): + """script() with ssh_to + ssh_config must build a flat argv (no nested lists). + + Regression: previously used cmd.append(["-F", ssh_config]) which embedded a + list inside the argv and broke subprocess.Popen when actually executing. + """ + + node = ExecuteNode(ssh_to="user@somehost", ssh_config="/tmp/some_ssh_config") + cmd_pipe = node.script(lines=["echo test"], pipe=True) + argv = cmd_pipe.items[0].cmd + + for i, arg in enumerate(argv): + self.assertIsInstance(arg, str, msg="argv[{}] should be str, got {!r}".format(i, arg)) + + # also verify the -F flag and value are present and adjacent + self.assertIn("-F", argv) + f_index = argv.index("-F") + self.assertEqual(argv[f_index + 1], "/tmp/some_ssh_config") + diff --git a/tests/test_externalfailures.py b/tests/test_externalfailures.py index 7c1dcd1..0347154 100644 --- a/tests/test_externalfailures.py +++ b/tests/test_externalfailures.py @@ -10,7 +10,6 @@ class TestExternalFailures(unittest2.TestCase): # generate a resumable state # NOTE: this generates two resumable test_target1/test_source1/fs1 and test_target1/test_source1/fs1/sub def generate_resume(self): - r = shelltest("zfs set compress=off test_source1 test_target1") # big change on source @@ -30,12 +29,11 @@ class TestExternalFailures(unittest2.TestCase): # resume initial backup def test_initial_resume(self): - # inital backup, leaves resume token with mocktime("20101111000000"): self.generate_resume() - # --test should resume and succeed + # test should resume and succeed with OutputIO() as buf: with redirect_stdout(buf): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test".split(" ")).run()) @@ -53,7 +51,7 @@ class TestExternalFailures(unittest2.TestCase): self.assertIn(": resuming", buf.getvalue()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 @@ -69,10 +67,10 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # resume incremental backup def test_incremental_resume(self): - # initial backup with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) # incremental backup leaves resume token with mocktime("20101111000001"): @@ -97,7 +95,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # did we really resume? self.assertIn(": resuming", buf.getvalue()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 @@ -113,9 +111,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 """) # generate an invalid resume token, and verify if its aborted automaticly - def test_initial_resumeabort(self): - - + def test_initial_abortedresume(self): # inital backup, leaves resume token with mocktime("20101111000000"): self.generate_resume() @@ -123,8 +119,6 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # remove corresponding source snapshot, so it becomes invalid shelltest("zfs destroy test_source1/fs1@test-20101111000000") - # NOTE: it can only abort the initial dataset if it has no subs - shelltest("zfs destroy test_target1/test_source1/fs1/sub; true") # --test try again, should abort old resume with mocktime("20101111000001"): @@ -134,7 +128,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 @@ -149,8 +143,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 """) # generate an invalid resume token, and verify if its aborted automaticly - def test_incremental_resumeabort(self): - + def test_incremental_abortedresume(self): # initial backup with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) @@ -166,11 +159,12 @@ test_target1/test_source2/fs2/sub@test-20101111000000 with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test".split(" ")).run()) + # try again, should abort old resume with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 @@ -187,7 +181,6 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # create a resume situation, where the other side doesnt want the snapshot anymore ( should abort resume ) def test_abort_unwanted_resume(self): - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) @@ -200,13 +193,14 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # incremental, doesnt want previous anymore with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --keep-target=0 --allow-empty --debug".split(" ")).run()) + "test test_target1 --no-progress --verbose --keep-target=0 --allow-empty --debug".split( + " ")).run()) print(buf.getvalue()) - self.assertIn("Aborting resume, we dont want that snapshot anymore.", buf.getvalue()) + self.assertIn("Aborting resume", buf.getvalue()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 @@ -222,7 +216,6 @@ test_target1/test_source2/fs2/sub@test-20101111000002 # test with empty snapshot list (this was a bug) def test_abort_resume_emptysnapshotlist(self): - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) @@ -240,15 +233,14 @@ test_target1/test_source2/fs2/sub@test-20101111000002 "test test_target1 --no-progress --verbose --no-snapshot".split( " ")).run()) - print(buf.getvalue()) - - self.assertIn("Aborting resume, its obsolete", buf.getvalue()) - + # NOTE: v3.4 doesnt abort the resume if there are no snapshots to send. + # print(buf.getvalue()) + # self.assertIn("Aborting resume, its obsolete", buf.getvalue()) def test_missing_common(self): - with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --no-bookmarks".split(" ")).run()) # remove common snapshot and leave nothing shelltest("zfs release zfs_autobackup:test test_source1/fs1@test-20101111000000") @@ -257,10 +249,9 @@ test_target1/test_source2/fs2/sub@test-20101111000002 with mocktime("20101111000001"): self.assertTrue(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - #UPDATE: offcourse the one thing that wasn't tested had a bug :( (in ExecuteNode.run()). + # UPDATE: offcourse the one thing that wasn't tested had a bug :( (in ExecuteNode.run()). def test_ignoretransfererrors(self): - - self.skipTest("Not sure how to implement a test for this without some serious hacking and patching.") + self.skipTest("Not sure how to implement a test for this without some serious hacking and patching.") # #recreate target pool without any features # # shelltest("zfs set compress=on test_source1; zpool destroy test_target1; zpool create test_target1 -o feature@project_quota=disabled /dev/ram2") @@ -268,7 +259,7 @@ test_target1/test_source2/fs2/sub@test-20101111000002 # with mocktime("20101111000000"): # self.assertFalse(ZfsAutobackup("test test_target1 --verbose --allow-empty --no-progress".split(" ")).run()) # -# r = shelltest("zfs list -H -o name -r -t all test_target1") +# r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") # # self.assertMultiLineEqual(r, """ # test_target1 diff --git a/tests/test_log.py b/tests/test_log.py index ba788e2..e4540df 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -24,6 +24,24 @@ class TestLog(unittest2.TestCase): print(list(buf.getvalue())) self.assertEqual(list(buf.getvalue()), ['\x1b', '[', '2', '2', 'm', ' ', ' ', 'v', 'e', 'r', 'b', 'o', 's', 'e', '\x1b', '[', '0', 'm', '\n', '\x1b', '[', '3', '2', 'm', '#', ' ', 'd', 'e', 'b', 'u', 'g', '\x1b', '[', '0', 'm', '\n', '\x1b', '[', '3', '1', 'm', '\x1b', '[', '1', 'm', '!', ' ', 'e', 'r', 'r', 'o', 'r', '\x1b', '[', '0', 'm', '\n']) + def test_progress_nocolor_no_crash(self): + """progress() + clear_progress() must not crash when color=False. + + Regression: clear_progress() used to do an unconditional `import colorama` + and call colorama.ansi.clear_line(), which fails if colorama isn't installed. + """ + + with OutputIO() as buf: + with redirect_stderr(buf): + l = LogConsole(show_verbose=False, show_debug=False, color=False) + l.progress("working") + l.progress("still working") # triggers clear_progress on the previous line + l.clear_progress() + # write something visible after clearing + l.error("done") + + self.assertIn("done", buf.getvalue()) + def test_nocolor(self): """test without color output""" diff --git a/tests/test_regressions.py b/tests/test_regressions.py index 7a5dac9..2bcfa82 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -1,4 +1,3 @@ - from basetest import * @@ -6,27 +5,28 @@ class TestZfsNode(unittest2.TestCase): def setUp(self): prepare_zpools() - self.longMessage=True + self.longMessage = True def test_keepsource0target10queuedsend(self): """Test if thinner doesnt destroy too much early on if there are no common snapshots YET. Issue #84""" + # old behaviour, without bookmarks with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send".split( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send --no-bookmarks".split( " ")).run()) with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send".split( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send --no-bookmarks".split( " ")).run()) with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty".split( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-bookmarks".split( " ")).run()) - r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 @@ -57,7 +57,6 @@ test_target1/test_source2/fs2/sub@test-20101111000001 test_target1/test_source2/fs2/sub@test-20101111000002 """) - def test_excludepaths(self): """Test issue #103""" @@ -70,8 +69,7 @@ test_target1/test_source2/fs2/sub@test-20101111000002 "test test_target1/target --no-progress --verbose --allow-empty".split( " ")).run()) - - r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 @@ -101,5 +99,3 @@ test_target1/target/test_target1/target_shouldnotbeexcluded@test-20101111000000 test_target1/target_shouldnotbeexcluded test_target1/target_shouldnotbeexcluded@test-20101111000000 """) - - diff --git a/tests/test_scaling.py b/tests/test_scaling.py index b93d9a8..b6abb21 100644 --- a/tests/test_scaling.py +++ b/tests/test_scaling.py @@ -2,14 +2,16 @@ from basetest import * from zfs_autobackup.ExecuteNode import ExecuteNode -run_orig=ExecuteNode.run -run_counter=0 +run_orig = ExecuteNode.run +run_counter = 0 + def run_count(*args, **kwargs): global run_counter - run_counter=run_counter+1 + run_counter = run_counter + 1 return (run_orig(*args, **kwargs)) + class TestZfsScaling(unittest2.TestCase): def setUp(self): @@ -19,83 +21,79 @@ class TestZfsScaling(unittest2.TestCase): def test_manysnapshots(self): """count the number of commands when there are many snapshots.""" - snapshot_count=100 + snapshot_count = 100 print("Creating many snapshots...") - s="" - for i in range(1970,1970+snapshot_count): - s=s+"zfs snapshot test_source1/fs1@test-{:04}1111000000;".format(i) + s = "" + for i in range(1970, 1970 + snapshot_count): + s = s + "zfs snapshot test_source1/fs1@test-{:04}1111000000;".format(i) shelltest(s) global run_counter - run_counter=0 - with patch.object(ExecuteNode,'run', run_count) as p: - + run_counter = 0 + with patch.object(ExecuteNode, 'run', run_count) as p: with mocktime("20101112000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --keep-source=10000 --keep-target=10000 --no-holds --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=10000 --keep-target=10000 --no-holds --allow-empty".split( + " ")).run()) - - #this triggers if you make a change with an impact of more than O(snapshot_count/2) - expected_runs=342 + # this triggers if you make a change with an impact of more than O(snapshot_count/2) + expected_runs = 549 print("EXPECTED RUNS: {}".format(expected_runs)) print("ACTUAL RUNS : {}".format(run_counter)) - self.assertLess(abs(run_counter-expected_runs), snapshot_count/2) - - - run_counter=0 - with patch.object(ExecuteNode,'run', run_count) as p: + self.assertLess(abs(run_counter - expected_runs), snapshot_count / 2) + run_counter = 0 + with patch.object(ExecuteNode, 'run', run_count) as p: with mocktime("20101112000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --keep-source=10000 --keep-target=10000 --no-holds --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=10000 --keep-target=10000 --no-holds --allow-empty".split( + " ")).run()) - - #this triggers if you make a change with a performance impact of more than O(snapshot_count/2) - expected_runs=47 + # this triggers if you make a change with a performance impact of more than O(snapshot_count/2) + expected_runs = 54 print("EXPECTED RUNS: {}".format(expected_runs)) print("ACTUAL RUNS : {}".format(run_counter)) - self.assertLess(abs(run_counter-expected_runs), snapshot_count/2) + self.assertLess(abs(run_counter - expected_runs), snapshot_count / 2) def test_manydatasets(self): """count the number of commands when when there are many datasets""" - dataset_count=100 + dataset_count = 100 print("Creating many datasets...") - s="" - for i in range(0,dataset_count): - s=s+"zfs create test_source1/fs1/{};".format(i) + s = "" + for i in range(0, dataset_count): + s = s + "zfs create test_source1/fs1/{};".format(i) shelltest(s) global run_counter - #first run - run_counter=0 - with patch.object(ExecuteNode,'run', run_count) as p: - + # first run + run_counter = 0 + with patch.object(ExecuteNode, 'run', run_count) as p: with mocktime("20101112000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) - - #this triggers if you make a change with an impact of more than O(snapshot_count/2)` - expected_runs=842 + # this triggers if you make a change with an impact of more than O(snapshot_count/2)` + expected_runs = 949 print("EXPECTED RUNS: {}".format(expected_runs)) print("ACTUAL RUNS: {}".format(run_counter)) - self.assertLess(abs(run_counter-expected_runs), dataset_count/2) - - - #second run, should have higher number of expected_runs - run_counter=0 - with patch.object(ExecuteNode,'run', run_count) as p: + self.assertLess(abs(run_counter - expected_runs), dataset_count / 2) + # second run, should have higher number of expected_runs + run_counter = 0 + with patch.object(ExecuteNode, 'run', run_count) as p: with mocktime("20101112000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) - - #this triggers if you make a change with a performance impact of more than O(snapshot_count/2) - expected_runs=1047 + # this triggers if you make a change with a performance impact of more than O(snapshot_count/2) + expected_runs = 1254 print("EXPECTED RUNS: {}".format(expected_runs)) print("ACTUAL RUNS: {}".format(run_counter)) - self.assertLess(abs(run_counter-expected_runs), dataset_count/2) + self.assertLess(abs(run_counter - expected_runs), dataset_count / 2) diff --git a/tests/test_sendrecvpipes.py b/tests/test_sendrecvpipes.py index c552540..bfd93e1 100644 --- a/tests/test_sendrecvpipes.py +++ b/tests/test_sendrecvpipes.py @@ -16,7 +16,8 @@ class TestSendRecvPipes(unittest2.TestCase): with self.subTest("local local pipe"): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", "--clear-mountpoint", + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", + "--clear-mountpoint", "--send-pipe=dd bs=1M", "--recv-pipe=dd bs=2M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -24,7 +25,7 @@ class TestSendRecvPipes(unittest2.TestCase): with self.subTest("remote local pipe"): with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", "--ssh-source=localhost", "--send-pipe=dd bs=1M", "--recv-pipe=dd bs=2M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -32,7 +33,7 @@ class TestSendRecvPipes(unittest2.TestCase): with self.subTest("local remote pipe"): with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", "--ssh-target=localhost", "--send-pipe=dd bs=1M", "--recv-pipe=dd bs=2M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -40,11 +41,11 @@ class TestSendRecvPipes(unittest2.TestCase): with self.subTest("remote remote pipe"): with mocktime("20101111000003"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", "--ssh-source=localhost", "--ssh-target=localhost", "--send-pipe=dd bs=1M", "--recv-pipe=dd bs=2M"]).run()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 @@ -74,7 +75,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 with self.subTest("compress " + compress): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--exclude-received", "--no-holds", "--no-progress", "--verbose", + ["test", "test_target1", "--no-holds", "--no-progress", "--verbose", "--compress=" + compress]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -85,14 +86,15 @@ test_target1/test_source2/fs2/sub@test-20101111000003 with self.subTest("local local pipe"): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", "--clear-mountpoint", "--buffer=1M"]).run()) + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", + "--clear-mountpoint", "--buffer=1M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") with self.subTest("remote local pipe"): with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--verbose", "--exclude-received", "--no-holds", + ["test", "test_target1", "--allow-empty", "--verbose", "--no-holds", "--no-progress", "--ssh-source=localhost", "--buffer=1M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -100,7 +102,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 with self.subTest("local remote pipe"): with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", "--ssh-target=localhost", "--buffer=1M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") @@ -108,10 +110,10 @@ test_target1/test_source2/fs2/sub@test-20101111000003 with self.subTest("remote remote pipe"): with mocktime("20101111000003"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", + ["test", "test_target1", "--allow-empty", "--no-holds", "--no-progress", "--ssh-source=localhost", "--ssh-target=localhost", "--buffer=1M"]).run()) - r = shelltest("zfs list -H -o name -r -t all test_target1") + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem test_target1") self.assertMultiLineEqual(r, """ test_target1 test_target1/test_source1 @@ -140,7 +142,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 start = time.time() with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - ["test", "test_target1", "--exclude-received", "--no-holds", "--no-progress", "--rate=50k"]).run()) + ["test", "test_target1", "--no-holds", "--no-progress", "--rate=50k"]).run()) # not a great way of verifying but it works. self.assertGreater(time.time() - start, 5) diff --git a/tests/test_thinner.py b/tests/test_thinner.py index 4425d3d..93919d7 100644 --- a/tests/test_thinner.py +++ b/tests/test_thinner.py @@ -2,6 +2,7 @@ from basetest import * import pprint from zfs_autobackup.Thinner import Thinner +from zfs_autobackup.ThinnerRule import ThinnerRule # randint is different in python 2 vs 3 randint_compat = lambda lo, hi: lo + int(random.random() * (hi + 1 - lo)) @@ -99,6 +100,51 @@ class TestThinner(unittest2.TestCase): self.assertEqual(result, ok) + def test_always_keep_only(self): + """Thinner('N') with no rules: keep at most N most-recent objects, regardless of age.""" + + thinner = Thinner("3") + + # fewer than N -> keep them all, remove nothing + few = [Thing(100), Thing(200)] + (keeps, removes) = thinner.thin(few, keep_objects=[], now=1000) + self.assertEqual(keeps, few) + self.assertEqual(removes, []) + + # exactly N -> keep them all, remove nothing + exact = [Thing(100), Thing(200), Thing(300)] + (keeps, removes) = thinner.thin(exact, keep_objects=[], now=1000) + self.assertEqual(keeps, exact) + self.assertEqual(removes, []) + + # more than N -> keep the last N, drop the rest + many = [Thing(100), Thing(200), Thing(300), Thing(400), Thing(500)] + (keeps, removes) = thinner.thin(many, keep_objects=[], now=1000) + self.assertEqual(keeps, many[-3:]) + self.assertEqual(removes, many[:-3]) + + def test_keep_objects_without_timestamp(self): + """objects with timestamp=None are always kept (treated as 'not ours').""" + + thinner = Thinner("1d1w") + no_ts = Thing(None) + old_ts = Thing(100) # well past the ttl + + (keeps, removes) = thinner.thin([no_ts, old_ts], keep_objects=[], now=10 ** 9) + self.assertIn(no_ts, keeps) + self.assertIn(old_ts, removes) + + def test_empty_thinner_keeps_everything(self): + """Thinner('') has no rules and always_keep=0 -> nothing is kept by rules, but timestamp=None objects still pass through.""" + + thinner = Thinner("") + with_ts = Thing(100) + without_ts = Thing(None) + + (keeps, removes) = thinner.thin([with_ts, without_ts], keep_objects=[], now=1000) + self.assertEqual(keeps, [without_ts]) + self.assertEqual(removes, [with_ts]) + def test_full(self): ok=['2022-03-09 01:56:23', '2023-01-03 10:53:16', diff --git a/tests/test_treehasher.py b/tests/test_treehasher.py index ea3cd04..e4648d6 100644 --- a/tests/test_treehasher.py +++ b/tests/test_treehasher.py @@ -15,10 +15,10 @@ class TestTreeHasher(unittest2.TestCase): def test_treehasher(self): shelltest("rm -rf /tmp/treehashertest; mkdir /tmp/treehashertest") - shelltest("cp tests/data/whole /tmp/treehashertest") + shelltest("cp data/whole /tmp/treehashertest") shelltest("mkdir /tmp/treehashertest/emptydir") shelltest("mkdir /tmp/treehashertest/dir") - shelltest("cp tests/data/whole_whole2_partial /tmp/treehashertest/dir") + shelltest("cp data/whole_whole2_partial /tmp/treehashertest/dir") # it should ignore these: shelltest("ln -s / /tmp/treehashertest/symlink") @@ -66,7 +66,7 @@ class TestTreeHasher(unittest2.TestCase): with self.subTest("Test mismatch"): generator = list(tree_hasher.generate("/tmp/treehashertest")) - shelltest("cp tests/data/whole2 /tmp/treehashertest/whole") + shelltest("cp data/whole2 /tmp/treehashertest/whole") self.assertEqual(list(tree_hasher.compare("/tmp/treehashertest", generator)), [('whole', diff --git a/tests/test_verify.py b/tests/test_verify.py index 5ecd402..c4164b7 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -69,7 +69,7 @@ class TestZfsVerify(unittest2.TestCase): self.assertFalse(ZfsAutoverify("test test_target1 --verbose --test".split(" ")).run()) with self.subTest("rsync, remote source and target. (not supported, all 6 fail)"): - self.assertEqual(6, ZfsAutoverify("test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --exclude-received --fs-compare=rsync".split(" ")).run()) + self.assertEqual(6, ZfsAutoverify("test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --fs-compare=rsync".split(" ")).run()) def runchecked(testname, command): with self.subTest(testname): @@ -83,20 +83,20 @@ class TestZfsVerify(unittest2.TestCase): self.assertRegex(buf.getvalue(), "bad_filesystem: FAILED:") self.assertRegex(buf.getvalue(), "bad_zvol: FAILED:") - runchecked("rsync, remote source", "test test_target1 --ssh-source=localhost --verbose --exclude-received --fs-compare=rsync") - runchecked("rsync, remote target", "test test_target1 --ssh-target=localhost --verbose --exclude-received --fs-compare=rsync") - runchecked("rsync, local", "test test_target1 --verbose --exclude-received --fs-compare=rsync") + runchecked("rsync, remote source", "test test_target1 --ssh-source=localhost --verbose --fs-compare=rsync") + runchecked("rsync, remote target", "test test_target1 --ssh-target=localhost --verbose --fs-compare=rsync") + runchecked("rsync, local", "test test_target1 --verbose --fs-compare=rsync") runchecked("tar, remote source and remote target", - "test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --exclude-received --fs-compare=find") + "test test_target1 --ssh-source=localhost --ssh-target=localhost --verbose --fs-compare=find") runchecked("tar, remote source", - "test test_target1 --ssh-source=localhost --verbose --exclude-received --fs-compare=find") + "test test_target1 --ssh-source=localhost --verbose --fs-compare=find") runchecked("tar, remote target", - "test test_target1 --ssh-target=localhost --verbose --exclude-received --fs-compare=find") - runchecked("tar, local", "test test_target1 --verbose --exclude-received --fs-compare=find") + "test test_target1 --ssh-target=localhost --verbose --fs-compare=find") + runchecked("tar, local", "test test_target1 --verbose --fs-compare=find") with self.subTest("no common snapshot"): #destroy common snapshot, now 3 should fail shelltest("zfs destroy test_source1/fs1/ok_zvol@test-20101111000000") - self.assertEqual(3, ZfsAutoverify("test test_target1 --verbose --exclude-received".split(" ")).run()) + self.assertEqual(3, ZfsAutoverify("test test_target1 --verbose ".split(" ")).run()) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index f05f15e..485b378 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -3,14 +3,15 @@ from zfs_autobackup.CmdPipe import CmdPipe from basetest import * import time -from zfs_autobackup.LogConsole import LogConsole +from zfs_autobackup.LogConsole import LogConsole +from zfs_autobackup.ZfsSnapshot import ZfsSnapshot class TestZfsAutobackup(unittest2.TestCase): def setUp(self): prepare_zpools() - self.longMessage=True + self.longMessage = True def test_invalidpars(self): @@ -18,7 +19,9 @@ class TestZfsAutobackup(unittest2.TestCase): with OutputIO() as buf: with redirect_stdout(buf): - self.assertEqual(ZfsAutobackup("test test_target1 --no-progress --resume --verbose --no-snapshot".split(" ")).run(), 0) + self.assertEqual( + ZfsAutobackup("test test_target1 --no-progress --resume --verbose --no-snapshot".split(" ")).run(), + 0) print(buf.getvalue()) self.assertIn("The --resume", buf.getvalue()) @@ -31,15 +34,14 @@ class TestZfsAutobackup(unittest2.TestCase): # correct message? self.assertIn("Please create this dataset", buf.getvalue()) - - def test_snapshotmode(self): + def test_snapshotmode(self): """test snapshot tool mode""" with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test --no-progress --verbose".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -54,27 +56,26 @@ test_source2/fs3/sub test_target1 """) - def test_defaults(self): - self.maxDiff=2000 + def test_defaults(self): + self.maxDiff = 2000 with self.subTest("no datasets selected"): with OutputIO() as buf: with redirect_stderr(buf): with mocktime("20101111000000"): - self.assertTrue(ZfsAutobackup("nonexisting test_target1 --verbose --debug --no-progress".split(" ")).run()) + self.assertTrue( + ZfsAutobackup("nonexisting test_target1 --verbose --debug --no-progress".split(" ")).run()) print(buf.getvalue()) - #correct message? + # correct message? self.assertIn("No source filesystems selected", buf.getvalue()) - with self.subTest("defaults with full verbose and debug"): - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --debug --no-progress".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -100,11 +101,11 @@ test_target1/test_source2/fs2/sub@test-20101111000000 with self.subTest("bare defaults, allow empty"): with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --no-progress".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1 --allow-empty --no-progress --no-bookmarks".split(" ")).run()) - - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -135,9 +136,8 @@ test_target1/test_source2/fs2/sub@test-20101111000001 """) with self.subTest("verify holds"): - - r=shelltest("zfs get -r userrefs test_source1 test_source2 test_target1") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs get -t filesystem,snapshot -r userrefs test_source1 test_source2 test_target1") + self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_source1 userrefs - - test_source1/fs1 userrefs - - @@ -168,56 +168,59 @@ test_target1/test_source2/fs2/sub@test-20101111000000 userrefs 0 - test_target1/test_source2/fs2/sub@test-20101111000001 userrefs 1 - """) - #make sure time handling is correctly. try to make snapshots a year appart and verify that only snapshots mostly 1y old are kept - #So in this case we only want to see 2 snapshots of 2011, and none of the 2010's anymore. + # make sure time handling is correctly. try to make snapshots a year appart and verify that only snapshots mostly 1y old are kept + # So in this case we only want to see 2 snapshots of 2011, and none of the 2010's anymore. with self.subTest("test time checking"): with mocktime("20111211000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --verbose --no-progress".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1 --allow-empty --verbose --no-progress".split(" ")).run()) - with mocktime("20111211000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --verbose --keep-source 1y1y --keep-target 1d1y --no-progress".split(" ")).run()) + with mocktime("20111212000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --allow-empty --verbose --keep-source 1d1y --keep-target 1d1y --no-progress".split( + " ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(""" test_source1 test_source1/fs1 test_source1/fs1@test-20111211000000 -test_source1/fs1@test-20111211000001 +test_source1/fs1@test-20111212000000 test_source1/fs1/sub test_source1/fs1/sub@test-20111211000000 -test_source1/fs1/sub@test-20111211000001 +test_source1/fs1/sub@test-20111212000000 test_source2 test_source2/fs2 test_source2/fs2/sub test_source2/fs2/sub@test-20111211000000 -test_source2/fs2/sub@test-20111211000001 +test_source2/fs2/sub@test-20111212000000 test_source2/fs3 test_source2/fs3/sub test_target1 test_target1/test_source1 test_target1/test_source1/fs1 test_target1/test_source1/fs1@test-20111211000000 -test_target1/test_source1/fs1@test-20111211000001 +test_target1/test_source1/fs1@test-20111212000000 test_target1/test_source1/fs1/sub test_target1/test_source1/fs1/sub@test-20111211000000 -test_target1/test_source1/fs1/sub@test-20111211000001 +test_target1/test_source1/fs1/sub@test-20111212000000 test_target1/test_source2 test_target1/test_source2/fs2 test_target1/test_source2/fs2/sub test_target1/test_source2/fs2/sub@test-20111211000000 -test_target1/test_source2/fs2/sub@test-20111211000001 -""") +test_target1/test_source2/fs2/sub@test-20111212000000 +""", r) - def test_ignore_othersnaphots(self): + def test_ignore_othersnaphots(self): - r=shelltest("zfs snapshot test_source1/fs1@othersimple") - r=shelltest("zfs snapshot test_source1/fs1@otherdate-20001111000000") + r = shelltest("zfs snapshot test_source1/fs1@othersimple") + r = shelltest("zfs snapshot test_source1/fs1@otherdate-20001111000000") with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@othersimple @@ -243,16 +246,17 @@ test_target1/test_source2/fs2/sub test_target1/test_source2/fs2/sub@test-20101111000000 """) - def test_othersnaphots(self): + def test_othersnaphots(self): - r=shelltest("zfs snapshot test_source1/fs1@othersimple") - r=shelltest("zfs snapshot test_source1/fs1@otherdate-20001111000000") + r = shelltest("zfs snapshot test_source1/fs1@othersimple") + r = shelltest("zfs snapshot test_source1/fs1@otherdate-20001111000000") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --other-snapshots".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1 --no-progress --verbose --other-snapshots".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@othersimple @@ -280,16 +284,15 @@ test_target1/test_source2/fs2/sub test_target1/test_source2/fs2/sub@test-20101111000000 """) - - def test_nosnapshot(self): + def test_nosnapshot(self): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --no-progress".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - #(only parents are created ) - #TODO: it probably shouldn't create these - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + # (only parents are created ) + # TODO: it probably shouldn't create these + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1/sub @@ -304,14 +307,13 @@ test_target1/test_source2 test_target1/test_source2/fs2 """) - - def test_nosend(self): + def test_nosend(self): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-send --no-progress".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -326,15 +328,15 @@ test_source2/fs3/sub test_target1 """) - - def test_ignorereplicated(self): - r=shelltest("zfs snapshot test_source1/fs1@otherreplication") + def test_ignorereplicated(self): + r = shelltest("zfs snapshot test_source1/fs1@otherreplication") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --ignore-replicated".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1 --no-progress --verbose --ignore-replicated".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@otherreplication @@ -357,13 +359,13 @@ test_target1/test_source2/fs2/sub test_target1/test_source2/fs2/sub@test-20101111000000 """) - def test_noholds(self): + def test_noholds(self): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-holds --no-progress".split(" ")).run()) - r=shelltest("zfs get -r userrefs test_source1 test_source2 test_target1") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs get -t filesystem,snapshot -r userrefs test_source1 test_source2 test_target1") + self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_source1 userrefs - - test_source1/fs1 userrefs - - @@ -388,14 +390,13 @@ test_target1/test_source2/fs2/sub userrefs - - test_target1/test_source2/fs2/sub@test-20101111000000 userrefs 0 - """) - - def test_strippath(self): + def test_strippath(self): with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --strip-path=1 --no-progress".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -418,27 +419,185 @@ test_target1/fs2/sub@test-20101111000000 """) def test_strippath_collision(self): - with self.assertRaisesRegexp(Exception,"collision"): + with self.assertRaisesRegexp(Exception, "collision"): ZfsAutobackup("test test_target1 --verbose --strip-path=2 --no-progress --debug".split(" ")).run() def test_strippath_toomuch(self): - with self.assertRaisesRegexp(Exception,"too much"): + with self.assertRaisesRegexp(Exception, "too much"): ZfsAutobackup("test test_target1 --verbose --strip-path=3 --no-progress --debug".split(" ")).run() - def test_clearrefres(self): + def test_property_format(self): + """--property-format selects datasets via a custom ZFS user-property name. - #on zfs utils 0.6.x -x isnt supported - r=shelltest("zfs recv -x bla test >/dev/null sys.exit(255) + with self.assertRaises(SystemExit) as cm: + ZfsAutobackup("test test_target1 --no-progress --snapshot-format {}__%Y_%m_%d".split(" ")) + self.assertEqual(cm.exception.code, 255) + + self.assertIn("Tag seperator", buf.getvalue()) + + def test_tagseperator_invalid(self): + """--tag-seperator must be non-empty and contain only characters valid in ZFS snapshot names ('_.: -').""" + + # empty string + with OutputIO() as buf: + with redirect_stderr(buf): + with self.assertRaises(SystemExit) as cm: + ZfsAutobackup("test test_target1 --no-progress --tag-seperator=".split(" ")) + self.assertEqual(cm.exception.code, 255) + self.assertIn("Invalid tag seperator", buf.getvalue()) + + # single invalid character + with OutputIO() as buf: + with redirect_stderr(buf): + with self.assertRaises(SystemExit) as cm: + ZfsAutobackup("test test_target1 --no-progress --tag-seperator=!".split(" ")) + self.assertEqual(cm.exception.code, 255) + self.assertIn("Invalid tag seperator", buf.getvalue()) + + # multi-char string where one character is invalid + with OutputIO() as buf: + with redirect_stderr(buf): + with self.assertRaises(SystemExit) as cm: + ZfsAutobackup("test test_target1 --no-progress --tag-seperator=_!".split(" ")) + self.assertEqual(cm.exception.code, 255) + self.assertIn("Invalid tag seperator", buf.getvalue()) + + def test_target_path_absolute_rejected(self): + """TARGET-PATH must be a ZFS path; a leading '/' is rejected (it's not a filesystem mountpoint).""" + + with OutputIO() as buf: + with redirect_stderr(buf): + with self.assertRaises(SystemExit) as cm: + ZfsAutobackup("test /test_target1 --no-progress".split(" ")) + self.assertEqual(cm.exception.code, 255) + self.assertIn("Target should not start with a /", buf.getvalue()) + + def test_tag_contains_seperator_rejected(self): + """--tag must not contain the tag-seperator (otherwise tagless_suffix parsing breaks).""" + + with OutputIO() as buf: + with redirect_stderr(buf): + with self.assertRaises(SystemExit) as cm: + # default tag-seperator is "__"; a tag containing "__" should be rejected. + ZfsAutobackup("test test_target1 --no-progress --tag=foo__bar".split(" ")) + self.assertEqual(cm.exception.code, 255) + self.assertIn("may not contain tag seperator", buf.getvalue()) + + def test_ignore_replicated_renamed_warning(self): + """--ignore-replicated is the old name; it should warn and be mapped to --exclude-unchanged.""" + + with mocktime("20101111000000"): + with OutputIO() as buf: + with redirect_stdout(buf): + # --ignore-replicated is hidden in --help but still accepted; run a no-op + # invocation so we can capture the rename warning without doing real work. + self.assertFalse(ZfsAutobackup( + "test --no-progress --verbose --no-snapshot --ignore-replicated".split(" ")).run()) + + self.assertIn("--ignore-replicated has been renamed", buf.getvalue()) + + def test_hold_format(self): + """--hold-format controls the zfs-hold name placed on snapshots.""" + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-bookmark --hold-format myhold_{}".split(" ")).run()) + + r = shelltest("zfs holds -H test_source1/fs1@test-20101111000000") + self.assertIn("myhold_test", r) + self.assertNotIn("zfs_autobackup:test", r) + + def test_clearrefres(self): + + # on zfs utils 0.6.x -x isnt supported + r = shelltest("zfs recv -x bla test >/dev/null /dev/null /dev/null >> .*minutes left.*") + self.assertRegex(buf.getvalue(), ".*>>> .*minutes left.*") diff --git a/tests/test_zfsautobackup31.py b/tests/test_zfsautobackup31.py index 17de12f..f6dd7b0 100644 --- a/tests/test_zfsautobackup31.py +++ b/tests/test_zfsautobackup31.py @@ -1,23 +1,25 @@ from basetest import * import time + class TestZfsAutobackup31(unittest2.TestCase): """various new 3.1 features""" def setUp(self): prepare_zpools() - self.longMessage=True + self.longMessage = True def test_no_thinning(self): - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --keep-target=0 --keep-source=0 --no-thinning".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --keep-target=0 --keep-source=0 --no-thinning".split( + " ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -47,7 +49,6 @@ test_target1/test_source2/fs2/sub@test-20101111000000 test_target1/test_source2/fs2/sub@test-20101111000001 """) - def test_re_replication(self): """test re-replication of something thats already a backup (new in v3.1-beta5)""" @@ -55,26 +56,48 @@ test_target1/test_source2/fs2/sub@test-20101111000001 shelltest("zfs create test_target1/b") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1/a --no-progress --verbose --debug".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1/a --no-progress --verbose --debug --allow-empty".split(" ")).run()) + + # NOTE: since v3.4 this changed. autobackup: properties are filtered. So its up to the admin to reset this property on the other side: + shelltest("zfs set autobackup:test=true test_target1/a") with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1/b --no-progress --verbose".split(" ")).run()) + self.assertFalse( + ZfsAutobackup("test test_target1/b --no-progress --verbose --allow-empty".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t snapshot test_target1") - #NOTE: it wont backup test_target1/a/test_source2/fs2/sub to test_target1/b since it doesnt have the zfs_autobackup property anymore. - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot test_target1") + # NOTE: it wont backup test_target1/a/test_source2/fs2/sub to test_target1/b since it doesnt have the zfs_autobackup property anymore. + self.assertMultiLineEqual(""" +test_target1/a@test-20101111000001 +test_target1/a/test_source1@test-20101111000001 test_target1/a/test_source1/fs1@test-20101111000000 +test_target1/a/test_source1/fs1@test-20101111000001 test_target1/a/test_source1/fs1/sub@test-20101111000000 +test_target1/a/test_source1/fs1/sub@test-20101111000001 +test_target1/a/test_source2@test-20101111000001 +test_target1/a/test_source2/fs2@test-20101111000001 test_target1/a/test_source2/fs2/sub@test-20101111000000 +test_target1/a/test_source2/fs2/sub@test-20101111000001 test_target1/b/test_source1/fs1@test-20101111000000 +test_target1/b/test_source1/fs1@test-20101111000001 test_target1/b/test_source1/fs1/sub@test-20101111000000 +test_target1/b/test_source1/fs1/sub@test-20101111000001 test_target1/b/test_source2/fs2/sub@test-20101111000000 +test_target1/b/test_source2/fs2/sub@test-20101111000001 +test_target1/b/test_target1/a@test-20101111000001 +test_target1/b/test_target1/a/test_source1@test-20101111000001 test_target1/b/test_target1/a/test_source1/fs1@test-20101111000000 +test_target1/b/test_target1/a/test_source1/fs1@test-20101111000001 test_target1/b/test_target1/a/test_source1/fs1/sub@test-20101111000000 -""") +test_target1/b/test_target1/a/test_source1/fs1/sub@test-20101111000001 +test_target1/b/test_target1/a/test_source2@test-20101111000001 +test_target1/b/test_target1/a/test_source2/fs2@test-20101111000001 +test_target1/b/test_target1/a/test_source2/fs2/sub@test-20101111000000 +test_target1/b/test_target1/a/test_source2/fs2/sub@test-20101111000001 +""", r) def test_zfs_compressed(self): - with mocktime("20101111000000"): self.assertFalse( ZfsAutobackup("test test_target1 --no-progress --verbose --debug --zfs-compressed".split(" ")).run()) @@ -86,19 +109,18 @@ test_target1/b/test_target1/a/test_source1/fs1/sub@test-20101111000000 with mocktime("20101111000000"): self.assertFalse( - ZfsAutobackup("test test_target1 --no-progress --verbose --debug --force --strip-path=1".split(" ")).run()) + ZfsAutobackup( + "test test_target1 --no-progress --verbose --debug --force --strip-path=1".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t snapshot test_target1") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot test_target1") + self.assertMultiLineEqual(r, """ test_target1@test-20101111000000 test_target1/fs1@test-20101111000000 test_target1/fs1/sub@test-20101111000000 test_target1/fs2/sub@test-20101111000000 """) - def test_exclude_unchanged(self): - shelltest("zfs snapshot -r test_source1@somesnapshot") with mocktime("20101111000000"): @@ -106,7 +128,7 @@ test_target1/fs2/sub@test-20101111000000 ZfsAutobackup( "test test_target1 --verbose --allow-empty --exclude-unchanged=1".split(" ")).run()) - #everything should be excluded, but should not return an error (see #190) + # everything should be excluded, but should not return an error (see #190) with mocktime("20101111000001"): self.assertFalse( ZfsAutobackup( @@ -116,4 +138,3 @@ test_target1/fs2/sub@test-20101111000000 self.assertMultiLineEqual(r, """ test_target1/test_source2/fs2/sub@test-20101111000000 """) - diff --git a/tests/test_zfsautobackup32.py b/tests/test_zfsautobackup32.py index 75771ce..08b68f3 100644 --- a/tests/test_zfsautobackup32.py +++ b/tests/test_zfsautobackup32.py @@ -1,29 +1,30 @@ from basetest import * + class TestZfsAutobackup32(unittest2.TestCase): """various new 3.2 features""" def setUp(self): prepare_zpools() - self.longMessage=True + self.longMessage = True def test_invalid_common_snapshot(self): - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - #create 2 snapshots with the same name, which are invalid as common snapshot + # create 2 snapshots with the same name, which are invalid as common snapshot shelltest("zfs snapshot test_source1/fs1@invalid") shelltest("zfs snapshot test_target1/test_source1/fs1@invalid") with mocktime("20101111000001"): - #try the old way (without guid checking), and fail: - self.assertEqual(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-guid-check".split(" ")).run(),1) - #new way should be ok: + # try the old way (without guid checking), and fail: + self.assertEqual(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --no-guid-check".split(" ")).run(), 1) + # new way should be ok: self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -56,23 +57,25 @@ test_target1/test_source2/fs2/sub@test-20101111000001 """) def test_invalid_common_snapshot_with_data(self): - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - #create 2 snapshots with the same name, which are invalid as common snapshot + # create 2 snapshots with the same name, which are invalid as common snapshot shelltest("zfs snapshot test_source1/fs1@invalid") shelltest("touch /test_target1/test_source1/fs1/shouldnotbeHere") shelltest("zfs snapshot test_target1/test_source1/fs1@invalid") with mocktime("20101111000001"): - #try the old way and fail: - self.assertEqual(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --destroy-incompatible --no-guid-check".split(" ")).run(),1) - #new way should be ok - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-snapshot --destroy-incompatible".split(" ")).run()) + # try the old way and fail: + self.assertEqual(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --destroy-incompatible --no-guid-check".split( + " ")).run(), 1) + # new way should be ok + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --no-snapshot --destroy-incompatible".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ test_source1 test_source1/fs1 test_source1/fs1@test-20101111000000 @@ -103,8 +106,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 test_target1/test_source2/fs2/sub@test-20101111000001 """) - - #check consistent mounting behaviour, see issue #112 + # check consistent mounting behaviour, see issue #112 def test_mount_consitency_mounted(self): """only filesystems that have canmount=on with a mountpoint should be mounted. """ @@ -113,35 +115,32 @@ test_target1/test_source2/fs2/sub@test-20101111000001 with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - r=shelltest("zfs mount |grep -o /test_target1.*") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs mount |grep -o /test_target1.*") + self.assertMultiLineEqual(r, """ /test_target1 /test_target1/test_source1/fs1 /test_target1/test_source1/fs1/sub /test_target1/test_source2/fs2/sub """) - def test_mount_consitency_unmounted(self): """only test_target1 should be mounted in this test""" shelltest("zfs create -V 10M test_source1/fs1/subvol") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --clear-mountpoint".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --clear-mountpoint".split(" ")).run()) - r=shelltest("zfs mount |grep -o /test_target1.*") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs mount |grep -o /test_target1.*") + self.assertMultiLineEqual(r, """ /test_target1 """) - - - def test_transfer_thinning(self): - # test pre/post/during transfer thinning and efficient transfer (no transerring of stuff that gets deleted on target) + # test pre/post/during transfer thinning and efficient transfer (no transerring of stuff that gets deleted on target) NO BOOKMARKS LEGACY MODE - #less output + # less output shelltest("zfs set autobackup:test2=true test_source1/fs1/sub") # nobody wants this one, will be destroyed before transferring (over a year ago) @@ -163,16 +162,17 @@ test_target1/test_source2/fs2/sub@test-20101111000001 with mocktime("20010202000000"): self.assertFalse(ZfsAutobackup("test2 --allow-empty".split(" ")).run()) - #will become common snapshot with OutputIO() as buf: with redirect_stdout(buf): + # now do thinning and transfer all at once with mocktime("20010203000000"): - self.assertFalse(ZfsAutobackup("--keep-source=1d10d --keep-target=1m10m --allow-empty --verbose --clear-mountpoint --other-snapshots test2 test_target1".split(" ")).run()) - + self.assertFalse(ZfsAutobackup( + "--keep-source=1d10d --keep-target=1m10m --allow-empty --verbose --clear-mountpoint --other-snapshots --no-bookmarks test2 test_target1".split( + " ")).run()) print(buf.getvalue()) self.assertIn( -""" + """ [Source] test_source1/fs1/sub@test2-20000101000000: Destroying [Source] test_source1/fs1/sub@test2-20010101000000: -> test_target1/test_source1/fs1/sub (new) [Source] test_source1/fs1/sub@other1: -> test_target1/test_source1/fs1/sub @@ -182,9 +182,8 @@ test_target1/test_source2/fs2/sub@test-20101111000001 [Source] test_source1/fs1/sub@test2-20010203000000: -> test_target1/test_source1/fs1/sub """, buf.getvalue()) - - r=shelltest("zfs list -H -o name -r -t snapshot test_source1 test_target1") - self.assertMultiLineEqual(r,""" + r = shelltest("zfs list -H -o name -r -t snapshot,bookmark test_source1 test_target1") + self.assertMultiLineEqual(r, """ test_source1/fs1/sub@other1 test_source1/fs1/sub@test2-20010201000000 test_source1/fs1/sub@other2 @@ -196,5 +195,3 @@ test_target1/test_source1/fs1/sub@test2-20010201000000 test_target1/test_source1/fs1/sub@other2 test_target1/test_source1/fs1/sub@test2-20010203000000 """) - - diff --git a/tests/test_zfsautobackup40.py b/tests/test_zfsautobackup40.py new file mode 100644 index 0000000..42d0f82 --- /dev/null +++ b/tests/test_zfsautobackup40.py @@ -0,0 +1,718 @@ +from basetest import * + + +class TestZfsAutobackup40(unittest2.TestCase): + """various new 4.0 features""" + + def setUp(self): + prepare_zpools() + self.longMessage = True + + def test_no_bookmark_source_support(self): + """test if everything is fine when source has no bookmark support (has no features at all even)""" + + subprocess.check_call("zpool destroy test_source1", shell=True) + subprocess.check_call("zpool create -d test_source1 /dev/ram0", shell=True) + shelltest("zpool get all test_source1") + subprocess.check_call("zfs create -p test_source1/fs1/sub", shell=True) # recreate with no features at all + subprocess.check_call("zfs set autobackup:test=true test_source1/fs1", shell=True) + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + # should fallback on holds on the source snapshot + r = shelltest("zfs holds test_source1/fs1@test-20101111000001") + self.assertIn("zfs_autobackup:test", r) + + def test_no_bookmark_target_support(self): + """test if everything is fine when target has no bookmark support (has no features at all even)""" + # NOTE: not sure if its ok if only the source supports bookmarks, so currently zfs-autobackup requires both sides to support bookmarks to enable it. + + subprocess.check_call("zpool destroy test_target1", shell=True) + subprocess.check_call("zpool create -d test_target1 /dev/ram2", shell=True) + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + # should fallback on holds on the source snapshot + r = shelltest("zfs holds test_source1/fs1@test-20101111000001") + self.assertIn("zfs_autobackup:test", r) + + def test_select_bookmark_or_snapshot(self): + """test if zfs autobackup chooses the most recent common matching dataset when there are both bookmarks and snapshots, some with the wrong GUID""" + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + target_guid = shelltest("zfs get -H -o value guid test_target1").strip() + + # destroy stuff and see if it still selects the correct ones + shelltest("zfs destroy test_source2/fs2/sub@test-20101111000001") + shelltest("zfs destroy test_source1/fs1/sub#test-20101111000001__" + target_guid) + + # replace bookmark with incorrect GUID, should fallback to snapshot + # NOTE: the incorrect bookmark should be ignored and not destroyed. + shelltest("zfs destroy test_source1/fs1#test-20101111000001__" + target_guid) + shelltest("zfs snapshot test_source1/fs1@wrong") + shelltest("zfs bookmark test_source1/fs1@wrong \#test-20101111000001__" + target_guid) + shelltest("zfs destroy test_source1/fs1@wrong") + + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + self.assertRegexpMatches(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000001 +test_source1/fs1@test-20101111000002 +test_source1/fs1#test-20101111000001__[0-9]* +test_source1/fs1#test-20101111000002__[0-9]* +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000001 +test_source1/fs1/sub@test-20101111000002 +test_source1/fs1/sub#test-20101111000002__[0-9]* +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000002 +test_source2/fs2/sub#test-20101111000002__[0-9]* +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@test-20101111000001 +test_target1/test_source1/fs1@test-20101111000002 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000001 +test_target1/test_source1/fs1/sub@test-20101111000002 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000001 +test_target1/test_source2/fs2/sub@test-20101111000002 +""") + + # while we're here, check that there are no holds on source common snapshot (since bookmarks replace holds on source side) + r = shelltest("zfs holds test_source2/fs2/sub@test-20101111000002") + self.assertNotIn("zfs_autobackup:test", r) + + def test_disable_bookmarks(self): + """test if we can disable it on an existing backup with bookmarks, with --no-bookmarks and get the old behaviour (holds on source)""" + + # first with bookmarks enabled + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + r = shelltest("zfs list -H -o name -r -t all test_source1") + self.assertRegexpMatches(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000001 +test_source1/fs1#test-20101111000001__[0-9]* +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000001 +test_source1/fs1/sub#test-20101111000001__[0-9]* +""") + + # disable it (this should not touch the old bookmark) + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --no-bookmarks".split(" ")).run()) + + r = shelltest("zfs list -H -o name -r -t all test_source1") + self.assertRegexpMatches(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000001 +test_source1/fs1@test-20101111000002 +test_source1/fs1#test-20101111000001__[0-9]* +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000001 +test_source1/fs1/sub@test-20101111000002 +test_source1/fs1/sub#test-20101111000001__[0-9]* +""") + + # re-enable (now the old bookmark should be still left alone) + with mocktime("20101111000003"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + r = shelltest("zfs list -H -o name -r -t all test_source1") + self.assertRegexpMatches(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000001 +test_source1/fs1@test-20101111000002 +test_source1/fs1@test-20101111000003 +test_source1/fs1#test-20101111000001__[0-9]* +test_source1/fs1#test-20101111000003__[0-9]* +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000001 +test_source1/fs1/sub@test-20101111000002 +test_source1/fs1/sub@test-20101111000003 +test_source1/fs1/sub#test-20101111000001__[0-9]* +test_source1/fs1/sub#test-20101111000003__[0-9]* +""") + + def test_tags(self): + with mocktime("20101111000001"): + self.assertFalse( + ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test1".split(" ")).run()) + + with mocktime("20101111000002"): + self.assertFalse( + ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --tag test2".split(" ")).run()) + + with mocktime("20101111000003"): + # make sure the thinner sees and cleans up the old snaphots that have a tag + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --keep-source=2 --keep-target=2".split( + " ")).run()) + + r = shelltest("zfs list -H -r -t snapshot -o name " + TEST_POOLS) + + self.assertMultiLineEqual(r, """ +test_source1/fs1@test-20101111000002__test2 +test_source1/fs1@test-20101111000003 +test_source1/fs1/sub@test-20101111000002__test2 +test_source1/fs1/sub@test-20101111000003 +test_source2/fs2/sub@test-20101111000002__test2 +test_source2/fs2/sub@test-20101111000003 +test_target1/test_source1/fs1@test-20101111000002__test2 +test_target1/test_source1/fs1@test-20101111000003 +test_target1/test_source1/fs1/sub@test-20101111000002__test2 +test_target1/test_source1/fs1/sub@test-20101111000003 +test_target1/test_source2/fs2/sub@test-20101111000002__test2 +test_target1/test_source2/fs2/sub@test-20101111000003 +""") + + def test_double_send_bookmark(self): + """test sending the same snaphots to 2 targets, and check if they each use their own bookmark and delete them correctly.""" + + shelltest("zfs create test_target1/a") + shelltest("zfs create test_target1/b") + + # full + with mocktime("20101111000001"): + self.assertFalse( + ZfsAutobackup( + "test test_target1/a --no-progress --verbose --allow-empty --debug --tag tagA".split(" ")).run()) + + # increment, should be from bookmark + with mocktime("20101111000002"): + self.assertFalse( + ZfsAutobackup( + "test test_target1/a --no-progress --verbose --allow-empty --tag tagB".split(" ")).run()) + + # to target b, now each has one full + two incrementals, which should be from their own bookmarks. + with mocktime("20101111000003"): + self.assertFalse( + ZfsAutobackup( + "test test_target1/b --no-progress --verbose --allow-empty".split( + " ")).run()) + + # result: + # for target a the bookmarks should be at 20101111000002, for target b the bookmarks should be at 20101111000003 + r = shelltest("zfs list -H -r -t all -o name " + TEST_POOLS) + + self.assertRegexpMatches(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000001__tagA +test_source1/fs1@test-20101111000002__tagB +test_source1/fs1@test-20101111000003 +test_source1/fs1#test-20101111000002__[0-9]* +test_source1/fs1#test-20101111000003__[0-9]* +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000001__tagA +test_source1/fs1/sub@test-20101111000002__tagB +test_source1/fs1/sub@test-20101111000003 +test_source1/fs1/sub#test-20101111000002__[0-9]* +test_source1/fs1/sub#test-20101111000003__[0-9]* +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000001__tagA +test_source2/fs2/sub@test-20101111000002__tagB +test_source2/fs2/sub@test-20101111000003 +test_source2/fs2/sub#test-20101111000002__[0-9]* +test_source2/fs2/sub#test-20101111000003__[0-9]* +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/a +test_target1/a/test_source1 +test_target1/a/test_source1/fs1 +test_target1/a/test_source1/fs1@test-20101111000001__tagA +test_target1/a/test_source1/fs1@test-20101111000002__tagB +test_target1/a/test_source1/fs1/sub +test_target1/a/test_source1/fs1/sub@test-20101111000001__tagA +test_target1/a/test_source1/fs1/sub@test-20101111000002__tagB +test_target1/a/test_source2 +test_target1/a/test_source2/fs2 +test_target1/a/test_source2/fs2/sub +test_target1/a/test_source2/fs2/sub@test-20101111000001__tagA +test_target1/a/test_source2/fs2/sub@test-20101111000002__tagB +test_target1/b +test_target1/b/test_source1 +test_target1/b/test_source1/fs1 +test_target1/b/test_source1/fs1@test-20101111000001__tagA +test_target1/b/test_source1/fs1@test-20101111000002__tagB +test_target1/b/test_source1/fs1@test-20101111000003 +test_target1/b/test_source1/fs1/sub +test_target1/b/test_source1/fs1/sub@test-20101111000001__tagA +test_target1/b/test_source1/fs1/sub@test-20101111000002__tagB +test_target1/b/test_source1/fs1/sub@test-20101111000003 +test_target1/b/test_source2 +test_target1/b/test_source2/fs2 +test_target1/b/test_source2/fs2/sub +test_target1/b/test_source2/fs2/sub@test-20101111000001__tagA +test_target1/b/test_source2/fs2/sub@test-20101111000002__tagB +test_target1/b/test_source2/fs2/sub@test-20101111000003 +""") + + def test_missing_common_bookmark(self): + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --no-holds".split(" ")).run()) + + # remove common bookmark + bookmark = shelltest("zfs list -H -o name -t bookmark test_source1/fs1").strip() + shelltest("zfs destroy " + bookmark) + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + def test_no_common_falls_back_to_incompatible(self): + """If no common snapshot/bookmark exists with target, target snapshots + must be reported as incompatible (instead of raising), so the user can + recover with --destroy-incompatible.""" + + # initial backup populates target and creates source bookmarks + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + # wipe every source snapshot and bookmark so nothing matches the target anymore + shelltest("zfs list -H -o name -t snapshot,bookmark -r test_source1 test_source2 " + "| xargs -rn1 zfs destroy") + + # without --destroy-incompatible the run must fail: target snapshots are in the way + with mocktime("20101111000001"): + self.assertTrue(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + # target snapshots from the initial backup must still be there (nothing destroyed yet) + r = shelltest("zfs list -H -o name -t snapshot test_target1/test_source1/fs1").strip() + self.assertIn("test-20101111000000", r) + + # with --destroy-incompatible -F the run succeeds: target is wiped and re-seeded + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --destroy-incompatible -F".split(" ")).run()) + + r = shelltest("zfs list -H -o name -t snapshot -r test_target1") + self.assertNotIn("test-20101111000000", r) + self.assertIn("test-20101111000002", r) + + def test_incompatible_dataset_test_mode(self): + """In --test mode, when no common snapshot exists, target dataset is + flagged as incompatible. Verify the output strings and that nothing + is actually changed.""" + + # initial backup + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + # wipe every source snapshot and bookmark so nothing matches target + shelltest("zfs list -H -o name -t snapshot,bookmark -r test_source1 test_source2 " + "| xargs -rn1 zfs destroy") + + # snapshot listing BEFORE running --test (to verify nothing changes) + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + # --test without --destroy-incompatible -F: must fail and report "Incompatible dataset!" + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000001"): + self.assertTrue(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + print(combined) + self.assertIn("Incompatible dataset", combined) + self.assertNotIn("STDERR", combined) + + + # --test WITH --destroy-incompatible -F: must succeed and warn "Overwriting incompatible dataset" + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --test --destroy-incompatible -F".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + self.assertIn("Overwriting incompatible dataset", combined) + self.assertNotIn("STDERR", combined) + + # --test must not have changed anything on disk + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + + def test_incompatible_snapshot_test_mode(self): + """In --test mode, when a common snapshot exists but extra incompatible + target snapshots are in the way, those must be reported as incompatible. + Verify the output string and that nothing is actually changed.""" + + # initial backup + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose".split(" ")).run()) + + # add an incompatible target snapshot (data written after common snapshot) + shelltest("touch /test_target1/test_source1/fs1/change.txt") + shelltest("zfs snapshot test_target1/test_source1/fs1@incompatible1") + + # snapshot listing BEFORE running --test (to verify nothing changes) + before = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + + # --test without --destroy-incompatible: must fail and warn about "Incompatible snapshot" + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000001"): + self.assertTrue(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --test".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + self.assertIn("Incompatible snapshot", combined) + self.assertNotIn("STDERR", combined) + + # --test WITH --destroy-incompatible: must succeed and still report "Incompatible snapshot" + with OutputIO() as buf_out, OutputIO() as buf_err: + with redirect_stdout(buf_out), redirect_stderr(buf_err): + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty --test --destroy-incompatible".split(" ")).run()) + combined = buf_out.getvalue() + buf_err.getvalue() + + self.assertIn("Incompatible snapshot", combined) + self.assertNotIn("STDERR", combined) + + # --test must not have changed anything on disk + after = shelltest("zfs list -H -o name -r -t snapshot " + TEST_POOLS) + self.assertMultiLineEqual(before, after) + + def test_keepsource0target10queuedsend_bookmarks(self): + """Test if thinner doesnt destroy too much early on if there are no common snapshots YET. Issue #84""" + # new behavior, with bookmarks. (will delete common snapshot, since there is a bookmark) + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send".split( + " ")).run()) + + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send".split( + " ")).run()) + + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty".split( + " ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1/sub +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@test-20101111000000 +test_target1/test_source1/fs1@test-20101111000001 +test_target1/test_source1/fs1@test-20101111000002 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000000 +test_target1/test_source1/fs1/sub@test-20101111000001 +test_target1/test_source1/fs1/sub@test-20101111000002 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000000 +test_target1/test_source2/fs2/sub@test-20101111000001 +test_target1/test_source2/fs2/sub@test-20101111000002 +""") + + def test_migrate_from_mismatching_bookmarkname(self): + """test migration from a bookmark with a mismatching name.""" + + shelltest("zfs snapshot test_source1/fs1@migrate1") + shelltest("zfs create test_target1/test_source1") + shelltest("zfs send test_source1/fs1@migrate1| zfs recv test_target1/test_source1/fs1") + + # make it so there is only the bookmark to migrate from, which a random non-matching name + shelltest("zfs bookmark test_source1/fs1@migrate1 \#randombookmarkname") + shelltest("zfs destroy test_source1/fs1@migrate1") + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test".split(" ")).run()) + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000000 +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000000 +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000000 +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@migrate1 +test_target1/test_source1/fs1@test-20101111000000 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000000 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000000 +""") + + def test_migrate_from_mismatching_snapshotname(self): + """test migration from a snapshot with a mismatching name.""" + + shelltest("zfs snapshot test_source1/fs1@migrate1") + shelltest("zfs create test_target1/test_source1") + shelltest("zfs send test_source1/fs1@migrate1| zfs recv test_target1/test_source1/fs1") + + # rename it so the names mismatch and guid matching is needed to resolve it + shelltest("zfs rename test_source1/fs1@migrate1 test_source1/fs1@randomsnapshotname") + + # testmode + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test".split(" ")).run()) + + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) + self.assertMultiLineEqual(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@randomsnapshotname +test_source1/fs1@test-20101111000000 +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000000 +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000000 +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@migrate1 +test_target1/test_source1/fs1@test-20101111000000 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000000 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000000 +""") + + def test_keep0_bookmarks(self): + """test if keep-source=0 and keep-target=0 only keeps a bookmark at the source. (contrary to legacy holds-mode)""" + + # backup mode + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=0".split( + " ")).run()) + + # snapshot mode, will delete 0 + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test --no-progress --verbose --keep-source=0 --keep-target=0 --allow-empty".split(" ")).run()) + + # snapshot mode, will detete 1 + with mocktime("20101111000002"): + self.assertFalse(ZfsAutobackup( + "test --no-progress --verbose --keep-source=0 --keep-target=0 --allow-empty".split(" ")).run()) + + # now we have only bookmark 0 and snapshot 2 at source + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem,bookmark " + TEST_POOLS) + self.assertRegexpMatches(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1@test-20101111000002 +test_source1/fs1#test-20101111000000__[0-9]* +test_source1/fs1/sub +test_source1/fs1/sub@test-20101111000002 +test_source1/fs1/sub#test-20101111000000__[0-9]* +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub@test-20101111000002 +test_source2/fs2/sub#test-20101111000000__[0-9]* +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@test-20101111000000 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000000 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000000 +""") + + # make another backup now the source only has bookmarks of 3 left + with mocktime("20101111000003"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=0 --allow-empty --debug".split( + " ")).run()) + + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem,bookmark " + TEST_POOLS) + self.assertRegexpMatches(r, """ +test_source1 +test_source1/fs1 +test_source1/fs1#test-20101111000003__[0-9]* +test_source1/fs1/sub +test_source1/fs1/sub#test-20101111000003__[0-9]* +test_source2 +test_source2/fs2 +test_source2/fs2/sub +test_source2/fs2/sub#test-20101111000003__[0-9]* +test_source2/fs3 +test_source2/fs3/sub +test_target1 +test_target1/test_source1 +test_target1/test_source1/fs1 +test_target1/test_source1/fs1@test-20101111000003 +test_target1/test_source1/fs1/sub +test_target1/test_source1/fs1/sub@test-20101111000003 +test_target1/test_source2 +test_target1/test_source2/fs2 +test_target1/test_source2/fs2/sub +test_target1/test_source2/fs2/sub@test-20101111000003 +""") + + def test_transfer_thinning_bookmarks(self): + # test pre/post/during transfer thinning and efficient transfer (no transerring of stuff that gets deleted on target) WITH BOOKMARK SUPPORT + + # less output + shelltest("zfs set autobackup:test2=true test_source1/fs1/sub") + + # nobody wants this one, will be destroyed before transferring (over a year ago) + with mocktime("20000101000000"): + self.assertFalse(ZfsAutobackup("test2 --allow-empty".split(" ")).run()) + + # only target wants this one (monthlys) + with mocktime("20010101000000"): + self.assertFalse(ZfsAutobackup("test2 --allow-empty".split(" ")).run()) + + # both want this one (dayly + monthly) + # other snapshots should influence the middle one that we actually want. + with mocktime("20010201000000"): + shelltest("zfs snapshot test_source1/fs1/sub@other1") + self.assertFalse(ZfsAutobackup("test2 --allow-empty".split(" ")).run()) + shelltest("zfs snapshot test_source1/fs1/sub@other2") + + # only source wants this one (dayly) + with mocktime("20010202000000"): + self.assertFalse(ZfsAutobackup("test2 --allow-empty".split(" ")).run()) + + with OutputIO() as buf: + with redirect_stdout(buf): + # now do thinning and transfer all at once + with mocktime("20010203000000"): + self.assertFalse(ZfsAutobackup( + "--keep-source=1d10d --keep-target=1m10m --allow-empty --verbose --clear-mountpoint --other-snapshots test2 test_target1".split( + " ")).run()) + + print(buf.getvalue()) + self.assertIn( + """ + [Source] test_source1/fs1/sub@test2-20000101000000: Destroying + [Source] test_source1/fs1/sub@test2-20010101000000: -> test_target1/test_source1/fs1/sub (new) + [Source] test_source1/fs1/sub@test2-20010101000000: Destroying + [Source] test_source1/fs1/sub@other1: -> test_target1/test_source1/fs1/sub + [Source] test_source1/fs1/sub@test2-20010201000000: -> test_target1/test_source1/fs1/sub + [Source] test_source1/fs1/sub@other2: -> test_target1/test_source1/fs1/sub + [Source] test_source1/fs1/sub@test2-20010203000000: -> test_target1/test_source1/fs1/sub +""", buf.getvalue()) + + # note: also important that it leaves only the last bookmark on the source. + r = shelltest("zfs list -H -o name -r -t snapshot,bookmark test_source1 test_target1") + self.assertRegexpMatches(r, """ +test_source1/fs1/sub@other1 +test_source1/fs1/sub@test2-20010201000000 +test_source1/fs1/sub@other2 +test_source1/fs1/sub@test2-20010202000000 +test_source1/fs1/sub@test2-20010203000000 +test_source1/fs1/sub#test2-20010203000000__[0-9]* +test_target1/test_source1/fs1/sub@test2-20010101000000 +test_target1/test_source1/fs1/sub@other1 +test_target1/test_source1/fs1/sub@test2-20010201000000 +test_target1/test_source1/fs1/sub@other2 +test_target1/test_source1/fs1/sub@test2-20010203000000 +""") + + def test_filter_autobackup_properties(self): + """All autobackup:* properties from the source must be filtered during + replication, and properties set directly on the target must not be + touched by subsequent backups.""" + + # an extra autobackup:* property on the source (besides the default autobackup:test=true on test_source1/fs1) + shelltest("zfs set autobackup:extra=sourceonly test_source1/fs1") + + # first backup, creates target hierarchy + with mocktime("20101111000000"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose".split(" ")).run()) + + # set autobackup:* properties directly on the target. + # autobackup:test deliberately collides with the source property to verify the source + # value does not overwrite the target on subsequent receives. + shelltest("zfs set autobackup:targetonly=targetvalue test_target1/test_source1/fs1") + shelltest("zfs set autobackup:test=false test_target1/test_source1/fs1/sub") + + properties = "autobackup:test,autobackup:extra,autobackup:targetonly" + before = shelltest( + "zfs get -H -o name,property,value,source -r -s local,received -t filesystem " + + properties + " test_target1") + + # second backup, must not change any autobackup:* property on the target + with mocktime("20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) + + after = shelltest( + "zfs get -H -o name,property,value,source -r -s local,received -t filesystem " + + properties + " test_target1") + + self.assertMultiLineEqual(before, after) + + # autobackup:extra was only set on the source and must not have leaked to the target + self.assertNotIn("sourceonly", after) + diff --git a/tests/test_zfscheck.py b/tests/test_zfscheck.py index cb239f3..a45fac1 100644 --- a/tests/test_zfscheck.py +++ b/tests/test_zfscheck.py @@ -9,7 +9,6 @@ class TestZfsCheck(unittest2.TestCase): def setUp(self): pass - def test_volume(self): if exists("/.dockerenv"): @@ -23,33 +22,36 @@ class TestZfsCheck(unittest2.TestCase): with self.subTest("Generate"): with OutputIO() as buf: with redirect_stdout(buf): - self.assertFalse(ZfsCheck("test_source1/vol@test".split(" "),print_arguments=False).run()) + self.assertFalse(ZfsCheck("test_source1/vol@test".split(" "), print_arguments=False).run()) print(buf.getvalue()) self.assertEqual("""0 2c2ceccb5ec5574f791d45b63c940cff20550f9a 1 2c2ceccb5ec5574f791d45b63c940cff20550f9a """, buf.getvalue()) - #store on disk for next step, add one error. + # store on disk for next step, add one error. with open("/tmp/testhashes", "w") as fh: - fh.write(buf.getvalue()+"1\t2c2ceccb5ec5574f791d45b63c940cff20550f9X") + fh.write(buf.getvalue() + "1\t2c2ceccb5ec5574f791d45b63c940cff20550f9X") with self.subTest("Compare"): with OutputIO() as buf: with redirect_stdout(buf): - self.assertEqual(1, ZfsCheck("test_source1/vol@test --check=/tmp/testhashes".split(" "),print_arguments=False).run()) + self.assertEqual(1, ZfsCheck("test_source1/vol@test --check=/tmp/testhashes".split(" "), + print_arguments=False).run()) print(buf.getvalue()) - self.assertEqual("Chunk 1 failed: 2c2ceccb5ec5574f791d45b63c940cff20550f9X 2c2ceccb5ec5574f791d45b63c940cff20550f9a\n", buf.getvalue()) + self.assertEqual( + "Chunk 1 failed: 2c2ceccb5ec5574f791d45b63c940cff20550f9X 2c2ceccb5ec5574f791d45b63c940cff20550f9a\n", + buf.getvalue()) def test_filesystem(self): prepare_zpools() - shelltest("cp tests/data/whole /test_source1/testfile") + shelltest("cp data/whole /test_source1/testfile") shelltest("mkdir /test_source1/emptydir") shelltest("mkdir /test_source1/dir") - shelltest("cp tests/data/whole2 /test_source1/dir/testfile") + shelltest("cp data/whole2 /test_source1/dir/testfile") - #it should ignore these: + # it should ignore these: shelltest("ln -s / /test_source1/symlink") shelltest("mknod /test_source1/c c 1 1") shelltest("mknod /test_source1/b b 1 1") @@ -67,24 +69,27 @@ class TestZfsCheck(unittest2.TestCase): dir/testfile 0 2e863f1fcccd6642e4e28453eba10d2d3f74d798 """, buf.getvalue()) - #store on disk for next step, add error + # store on disk for next step, add error with open("/tmp/testhashes", "w") as fh: - fh.write(buf.getvalue()+"dir/testfile 0 2e863f1fcccd6642e4e28453eba10d2d3f74d79X") + fh.write(buf.getvalue() + "dir/testfile 0 2e863f1fcccd6642e4e28453eba10d2d3f74d79X") with self.subTest("Compare"): with OutputIO() as buf: with redirect_stdout(buf): - self.assertEqual(1, ZfsCheck("test_source1@test --check=/tmp/testhashes".split(" "),print_arguments=False).run()) + self.assertEqual(1, ZfsCheck("test_source1@test --check=/tmp/testhashes".split(" "), + print_arguments=False).run()) print(buf.getvalue()) - self.assertEqual("dir/testfile: Chunk 0 failed: 2e863f1fcccd6642e4e28453eba10d2d3f74d79X 2e863f1fcccd6642e4e28453eba10d2d3f74d798\n", buf.getvalue()) + self.assertEqual( + "dir/testfile: Chunk 0 failed: 2e863f1fcccd6642e4e28453eba10d2d3f74d79X 2e863f1fcccd6642e4e28453eba10d2d3f74d798\n", + buf.getvalue()) def test_file(self): with self.subTest("Generate"): with OutputIO() as buf: with redirect_stdout(buf): - self.assertFalse(ZfsCheck("tests/data/whole".split(" "), print_arguments=False).run()) + self.assertFalse(ZfsCheck("data/whole".split(" "), print_arguments=False).run()) print(buf.getvalue()) self.assertEqual("""0 3c0bf91170d873b8e327d3bafb6bc074580d11b7 @@ -92,22 +97,25 @@ dir/testfile 0 2e863f1fcccd6642e4e28453eba10d2d3f74d798 # store on disk for next step, add error with open("/tmp/testhashes", "w") as fh: - fh.write(buf.getvalue()+"0 3c0bf91170d873b8e327d3bafb6bc074580d11bX") + fh.write(buf.getvalue() + "0 3c0bf91170d873b8e327d3bafb6bc074580d11bX") with self.subTest("Compare"): with OutputIO() as buf: with redirect_stdout(buf): - self.assertEqual(1,ZfsCheck("tests/data/whole --check=/tmp/testhashes".split(" "), print_arguments=False).run()) + self.assertEqual(1, ZfsCheck("data/whole --check=/tmp/testhashes".split(" "), + print_arguments=False).run()) print(buf.getvalue()) - self.assertEqual("Chunk 0 failed: 3c0bf91170d873b8e327d3bafb6bc074580d11bX 3c0bf91170d873b8e327d3bafb6bc074580d11b7\n", buf.getvalue()) + self.assertEqual( + "Chunk 0 failed: 3c0bf91170d873b8e327d3bafb6bc074580d11bX 3c0bf91170d873b8e327d3bafb6bc074580d11b7\n", + buf.getvalue()) def test_tree(self): shelltest("rm -rf /tmp/testtree; mkdir /tmp/testtree") - shelltest("cp tests/data/whole /tmp/testtree") - shelltest("cp tests/data/whole_whole2 /tmp/testtree") - shelltest("cp tests/data/whole2 /tmp/testtree") - shelltest("cp tests/data/partial /tmp/testtree") - shelltest("cp tests/data/whole_whole2_partial /tmp/testtree") + shelltest("cp data/whole /tmp/testtree") + shelltest("cp data/whole_whole2 /tmp/testtree") + shelltest("cp data/whole2 /tmp/testtree") + shelltest("cp data/partial /tmp/testtree") + shelltest("cp data/whole_whole2_partial /tmp/testtree") #################################### with self.subTest("Generate, skip 1"): @@ -115,13 +123,12 @@ dir/testfile 0 2e863f1fcccd6642e4e28453eba10d2d3f74d798 with redirect_stdout(buf): self.assertFalse(ZfsCheck("/tmp/testtree --skip=1".split(" "), print_arguments=False).run()) - #since order varies, just check count (there is one empty line for some reason, only when testing like this) + # since order varies, just check count (there is one empty line for some reason, only when testing like this) print(buf.getvalue().split("\n")) - self.assertEqual(len(buf.getvalue().split("\n")),4) + self.assertEqual(len(buf.getvalue().split("\n")), 4) ###################################### with self.subTest("Compare, all incorrect, skip 1"): - # store on disk for next step, add error with open("/tmp/testhashes", "w") as fh: fh.write(""" @@ -134,13 +141,14 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bdX with OutputIO() as buf: with redirect_stdout(buf): - self.assertEqual(ZfsCheck("/tmp/testtree --check=/tmp/testhashes --skip=1".split(" "), print_arguments=False).run(), 3) + self.assertEqual(ZfsCheck("/tmp/testtree --check=/tmp/testhashes --skip=1".split(" "), + print_arguments=False).run(), 3) print(buf.getvalue()) self.assertMultiLineEqual("""partial: Chunk 0 failed: 642027d63bb0afd7e0ba197f2c66ad03e3d70deX 642027d63bb0afd7e0ba197f2c66ad03e3d70de1 whole2: Chunk 0 failed: 2e863f1fcccd6642e4e28453eba10d2d3f74d79X 2e863f1fcccd6642e4e28453eba10d2d3f74d798 whole_whole2_partial: Chunk 0 failed: 309ffffba2e1977d12f3b7469971f30d28b94bdX 309ffffba2e1977d12f3b7469971f30d28b94bd8 -""",buf.getvalue()) +""", buf.getvalue()) #################################### with self.subTest("Generate"): @@ -148,10 +156,10 @@ whole_whole2_partial: Chunk 0 failed: 309ffffba2e1977d12f3b7469971f30d28b94bdX 3 with redirect_stdout(buf): self.assertFalse(ZfsCheck("/tmp/testtree".split(" "), print_arguments=False).run()) - #file order on disk can vary, so sort it.. - sorted=buf.getvalue().split("\n") + # file order on disk can vary, so sort it.. + sorted = buf.getvalue().split("\n") sorted.sort() - sorted="\n".join(sorted)+"\n" + sorted = "\n".join(sorted) + "\n" print(sorted) self.assertEqual(""" @@ -181,14 +189,14 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bd8 """test if stuff is cleaned up correctly, in debugging mode , when a pipe breaks. """ prepare_zpools() - shelltest("cp tests/data/whole /test_source1/testfile") + shelltest("cp data/whole /test_source1/testfile") shelltest("zfs snapshot test_source1@test") - #breaks pipe when head exists - #important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling - shelltest("python -m zfs_autobackup.ZfsCheck test_source1@test --debug | head -n1") + # breaks pipe when head exists + # important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling + shelltest("cd ..;python3 -m zfs_autobackup.ZfsCheck test_source1@test --debug | head -n1") - #should NOT be mounted anymore if cleanup went ok: + # should NOT be mounted anymore if cleanup went ok: self.assertNotRegex(shelltest("mount"), "test_source1@test") def test_brokenpipe_cleanup_volume(self): @@ -199,12 +207,12 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bd8 shelltest("zfs create -V200M test_source1/vol") shelltest("zfs snapshot test_source1/vol@test") - #breaks pipe when grep exists: - #important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling - shelltest("python -m zfs_autobackup.ZfsCheck test_source1/vol@test --debug| grep -m1 'Hashing file'") + # breaks pipe when grep exists: + # important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling + shelltest("cd ..;python3 -m zfs_autobackup.ZfsCheck test_source1/vol@test --debug| grep -m1 'Hashing file'") # time.sleep(1) - r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem,volume " + TEST_POOLS) self.assertMultiLineEqual(""" test_source1 test_source1/fs1 @@ -217,7 +225,4 @@ test_source2/fs2/sub test_source2/fs3 test_source2/fs3/sub test_target1 -""",r ) - - - +""", r) diff --git a/tests/test_zfsnode.py b/tests/test_zfsnode.py index 3cbff5b..655708b 100644 --- a/tests/test_zfsnode.py +++ b/tests/test_zfsnode.py @@ -1,6 +1,7 @@ from basetest import * from zfs_autobackup.LogStub import LogStub from zfs_autobackup.ExecuteNode import ExecuteError +from zfs_autobackup.ZfsSnapshot import ZfsSnapshot class TestZfsNode(unittest2.TestCase): @@ -12,13 +13,16 @@ class TestZfsNode(unittest2.TestCase): def test_consistent_snapshot(self): logger = LogStub() description = "[Source]" - node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description, debug_output=False, ssh_to=None, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') with self.subTest("first snapshot"): - (selected_datasets, excluded_datasets)=node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, - exclude_unchanged=0) + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], + exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-20101111000001", 100000) - r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertEqual(r, """ test_source1 test_source1/fs1 @@ -35,10 +39,11 @@ test_target1 """) with self.subTest("second snapshot, no changes, no snapshot"): - (selected_datasets, excluded_datasets)=node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, - exclude_unchanged=0) + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], + exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-20101111000002", 1) - r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertEqual(r, """ test_source1 test_source1/fs1 @@ -55,9 +60,11 @@ test_target1 """) with self.subTest("second snapshot, no changes, empty snapshot"): - (selected_datasets, excluded_datasets) =node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, exclude_unchanged=0) + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], + exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-20101111000002", 0) - r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) + r = shelltest("zfs list -H -o name -r -t snapshot,filesystem " + TEST_POOLS) self.assertEqual(r, """ test_source1 test_source1/fs1 @@ -79,12 +86,16 @@ test_target1 def test_consistent_snapshot_prepostcmds(self): logger = LogStub() description = "[Source]" - node = ZfsNode(utc=False, snapshot_time_format="test", hold_name="test", logger=logger, description=description, debug_output=True) + node = ZfsNode(utc=False, snapshot_time_format="test", hold_name="test", logger=logger, description=description, + debug_output=True, ssh_to=None, ssh_config=None, exclude_snapshot_patterns=None, thinner=None, + readonly=False, tag_seperator='_') with self.subTest("Test if all cmds are executed correctly (no failures)"): with OutputIO() as buf: with redirect_stdout(buf): - (selected_datasets, excluded_datasets) =node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, exclude_unchanged=0) + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], + exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-1", 0, pre_snapshot_cmds=["echo pre1", "echo pre2"], @@ -96,12 +107,14 @@ test_target1 self.assertIn("STDOUT > post1", buf.getvalue()) self.assertIn("STDOUT > post2", buf.getvalue()) - - with self.subTest("Failure in the middle, only pre1 and both post1 and post2 should be executed, no snapshot should be attempted"): + with self.subTest( + "Failure in the middle, only pre1 and both post1 and post2 should be executed, no snapshot should be attempted"): with OutputIO() as buf: with redirect_stdout(buf): with self.assertRaises(ExecuteError): - (selected_datasets, excluded_datasets) =node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, exclude_unchanged=0) + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], + exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-1", 0, pre_snapshot_cmds=["echo pre1", "false", "echo pre2"], @@ -118,8 +131,10 @@ test_target1 with OutputIO() as buf: with redirect_stdout(buf): with self.assertRaises(ExecuteError): - #same snapshot name as before so it fails - (selected_datasets, excluded_datasets) =node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, exclude_unchanged=0) + # same snapshot name as before so it fails + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], + exclude_unchanged=0) node.consistent_snapshot(selected_datasets, "test-1", 0, pre_snapshot_cmds=["echo pre1", "echo pre2"], @@ -136,20 +151,22 @@ test_target1 # Assert that timestamps keep relative order both for utc and for localtime logger = LogStub() description = "[Source]" - node_local = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) - node_utc = ZfsNode(utc=True, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) + node_local = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description, debug_output=False, ssh_to=None, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') + node_utc = ZfsNode(utc=True, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description, debug_output=False, ssh_to=None, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') for node in [node_local, node_utc]: with self.subTest("timestamp ordering " + ("utc" if node == node_utc else "localtime")): - dataset_a = ZfsDataset(node,"test_source1@test-20101111000001") - dataset_b = ZfsDataset(node,"test_source1@test-20101111000002") - dataset_c = ZfsDataset(node,"test_source1@test-20240101020202") + dataset_a = ZfsSnapshot(node, "test_source1@test-20101111000001") + dataset_b = ZfsSnapshot(node, "test_source1@test-20101111000002") + dataset_c = ZfsSnapshot(node, "test_source1@test-20240101020202") self.assertGreater(dataset_b.timestamp, dataset_a.timestamp) self.assertGreater(dataset_c.timestamp, dataset_b.timestamp) - def test_getselected(self): - # should be excluded by property shelltest("zfs create test_source1/fs1/subexcluded") shelltest("zfs set autobackup:test=false test_source1/fs1/subexcluded") @@ -165,9 +182,12 @@ test_target1 logger = LogStub() description = "[Source]" - node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) - (selected_datasets, excluded_datasets)=node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, - exclude_unchanged=1) + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description, debug_output=False, ssh_to=None, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') + (selected_datasets, excluded_datasets) = node.selected_datasets(property_name="autobackup:test", + exclude_paths=[], + exclude_unchanged=1) s = pformat(selected_datasets) print(s) @@ -177,11 +197,12 @@ test_target1 (local): test_source2/fs2/sub, (local): test_source1/fs1/onlyparent]""") - def test_validcommand(self): logger = LogStub() description = "[Source]" - node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description, debug_output=False, ssh_to=None, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') with self.subTest("test invalid option"): self.assertFalse(node.valid_command(["zfs", "send", "--invalid-option", "nonexisting"])) @@ -191,7 +212,9 @@ test_target1 def test_supportedsendoptions(self): logger = LogStub() description = "[Source]" - node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description, debug_output=False, ssh_to=None, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') # -D propably always supported self.assertGreater(len(node.supported_send_options), 0) @@ -199,7 +222,9 @@ test_target1 logger = LogStub() description = "[Source]" # NOTE: this could hang via ssh if we dont close filehandles properly. (which was a previous bug) - node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description, ssh_to='localhost') + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", + logger=logger, description=description, ssh_to='localhost', debug_output=False, ssh_config=None, + exclude_snapshot_patterns=None, thinner=None, readonly=False, tag_seperator='_') self.assertIsInstance(node.supported_recv_options, list) diff --git a/tests/tests b/tests/tests deleted file mode 120000 index 945c9b4..0000000 --- a/tests/tests +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/zfs_autobackup/BlockHasher.py b/zfs_autobackup/BlockHasher.py index 0a37c90..0e82f59 100644 --- a/zfs_autobackup/BlockHasher.py +++ b/zfs_autobackup/BlockHasher.py @@ -19,15 +19,14 @@ class BlockHasher(): def __init__(self, count=10000, bs=4096, hash_class=hashlib.sha1, skip=0): self.count = count self.bs = bs - self.chunk_size=bs*count + self.chunk_size = bs * count self.hash_class = hash_class # self.coverage=coverage - self.skip=skip - self._skip_count=0 - - self.stats_total_bytes=0 + self.skip = skip + self._skip_count = 0 + self.stats_total_bytes = 0 def _seek_next_chunk(self, fh, fsize): """seek fh to next chunk and update skip counter. @@ -37,8 +36,8 @@ class BlockHasher(): """ - #ignore rempty files - if fsize==0: + # ignore rempty files + if fsize == 0: return False # need to skip chunks? @@ -53,7 +52,7 @@ class BlockHasher(): # seek to next chunk, reset skip count fh.seek(self.chunk_size * self._skip_count, os.SEEK_CUR) self._skip_count = self.skip - return fh.tell()//self.chunk_size + return fh.tell() // self.chunk_size else: # should read this chunk, reset skip count self._skip_count = self.skip @@ -67,24 +66,23 @@ class BlockHasher(): yields nothing for empty files. """ - with open(fname, "rb") as fh: fh.seek(0, os.SEEK_END) - fsize=fh.tell() + fsize = fh.tell() fh.seek(0) - while fh.tell() {}".format(exit_code)) if (valid_exitcodes != []) and (exit_code not in valid_exitcodes): - self.error("Command \"{}\" returned exit code {} (valid codes: {})".format(cmd_item, exit_code, valid_exitcodes)) + self.error("Command \"{}\" returned exit code {} (valid codes: {})".format(cmd_item, exit_code, + valid_exitcodes)) return False return True @@ -149,7 +153,7 @@ class ExecuteNode(LogStub): if pipe: # dont specify output handler, so it will get piped to next process - stdout_handler=None + stdout_handler = None else: # handle output manually, dont pipe it def stdout_handler(line): @@ -160,7 +164,8 @@ class ExecuteNode(LogStub): self._parse_stdout(line) # add shell command and handlers to pipe - cmd_item=CmdItem(cmd=self._shell_cmd(cmd, cwd), readonly=readonly, stderr_handler=stderr_handler, exit_handler=exit_handler, shell=self.is_local(), stdout_handler=stdout_handler) + cmd_item = CmdItem(cmd=self._shell_cmd(cmd, cwd), readonly=readonly, stderr_handler=stderr_handler, + exit_handler=exit_handler, shell=self.is_local(), stdout_handler=stdout_handler) cmd_pipe.add(cmd_item) # return CmdPipe instead of executing? @@ -174,7 +179,7 @@ class ExecuteNode(LogStub): # execute and calls handlers in CmdPipe if not cmd_pipe.execute(): - raise(ExecuteError("Last command returned error")) + raise (ExecuteError("Last command returned error")) if return_all: return output_lines, error_lines, cmd_item.process and cmd_item.process.returncode @@ -183,7 +188,8 @@ class ExecuteNode(LogStub): else: return output_lines - def script(self, lines, inp=None, stdout_handler=None, stderr_handler=None, exit_handler=None, valid_exitcodes=None, readonly=False, hide_errors=False, pipe=False): + def script(self, lines, inp=None, stdout_handler=None, stderr_handler=None, exit_handler=None, valid_exitcodes=None, + readonly=False, hide_errors=False, pipe=False): """Run a multiline script on the node. This is much more low level than run() and allows for finer grained control. @@ -212,14 +218,14 @@ class ExecuteNode(LogStub): # add stuff to existing pipe cmd_pipe = inp - internal_stdout_handler=None + internal_stdout_handler = None if stdout_handler is not None: if self.debug_output: def internal_stdout_handler(line): self.debug("STDOUT > " + line.rstrip()) stdout_handler(line) else: - internal_stdout_handler=stdout_handler + internal_stdout_handler = stdout_handler def internal_stderr_handler(line): self._parse_stderr(line, hide_errors) @@ -243,16 +249,16 @@ class ExecuteNode(LogStub): return True - #build command - cmd=[] + # build command + cmd = [] - #add remote shell + # add remote shell if not self.is_local(): - #note: dont escape this part (executed directly without shell) + # note: dont escape this part (executed directly without shell) cmd.append("ssh") if self.ssh_config is not None: - cmd.append(["-F", self.ssh_config]) + cmd.extend(["-F", self.ssh_config]) cmd.append(self.ssh_to) @@ -260,7 +266,9 @@ class ExecuteNode(LogStub): cmd.append("\n".join(lines)) # add shell command and handlers to pipe - cmd_item=CmdItem(cmd=cmd, readonly=readonly, stderr_handler=internal_stderr_handler, exit_handler=internal_exit_handler, stdout_handler=internal_stdout_handler, shell=self.is_local()) + cmd_item = CmdItem(cmd=cmd, readonly=readonly, stderr_handler=internal_stderr_handler, + exit_handler=internal_exit_handler, stdout_handler=internal_stdout_handler, + shell=self.is_local()) cmd_pipe.add(cmd_item) self.debug("SCRIPT > {}".format(cmd_pipe)) diff --git a/zfs_autobackup/LogConsole.py b/zfs_autobackup/LogConsole.py index a7e799a..acd905f 100644 --- a/zfs_autobackup/LogConsole.py +++ b/zfs_autobackup/LogConsole.py @@ -2,28 +2,34 @@ from __future__ import print_function import sys +import time + +try: + import colorama + _COLORAMA_AVAILABLE = True +except ImportError: + _COLORAMA_AVAILABLE = False + class LogConsole: """Log-class that outputs to console, adding colors if needed""" + # ANSI escape used as a fallback when colorama isn't available + _CLEAR_LINE = "\033[2K\r" + def __init__(self, show_debug, show_verbose, color): self.last_log = "" self.show_debug = show_debug self.show_verbose = show_verbose - self._progress_uncleared=False + self.show_progress=True - if color: - # try to use color, failback if colorama not available - self.colorama=False - try: - import colorama - global colorama - self.colorama = True - except ImportError: - pass + self._progress_uncleared = False - else: - self.colorama=False + self.colorama = color and _COLORAMA_AVAILABLE + + self._failed=None + self._current=None + self._total=None def error(self, txt): self.clear_progress() @@ -59,16 +65,43 @@ class LogConsole: print("# " + txt) sys.stdout.flush() - def progress(self, txt): + def progress(self, txt, current=None, total=None, failed=None): + if not self.show_progress: + return + + if current is not None: + self._current=current + + if total is not None: + self._total=total + + if failed is not None: + self._failed=failed + """print progress output to stderr (stays on same line)""" self.clear_progress() - self._progress_uncleared=True - print(">>> {}\r".format(txt), end='', file=sys.stderr) + self._progress_uncleared = True + + if self._current is None: + line=f">>> {txt}" + else: + if self._failed: + line=f">>> [{self._current}/{self._total}] {txt} ({self._failed} failed)" + else: + line=f">>> [{self._current}/{self._total}] {txt}" + + print(line+"\r", end='', file=sys.stderr) + sys.stderr.flush() def clear_progress(self): + if not self.show_progress: + return + if self._progress_uncleared: - import colorama - print(colorama.ansi.clear_line(), end='', file=sys.stderr) - # sys.stderr.flush() - self._progress_uncleared=False + if self.colorama: + print(colorama.ansi.clear_line(), end='', file=sys.stderr) + else: + print(self._CLEAR_LINE, end='', file=sys.stderr) + self._progress_uncleared = False + sys.stderr.flush() diff --git a/zfs_autobackup/LogStub.py b/zfs_autobackup/LogStub.py index 5d593c1..0d00e31 100644 --- a/zfs_autobackup/LogStub.py +++ b/zfs_autobackup/LogStub.py @@ -1,5 +1,5 @@ -#Used for baseclasses that dont implement their own logging (Like ExecuteNode) -#Usually logging is implemented in subclasses (Like ZfsNode thats a subclass of ExecuteNode), but for regression testing its nice to have these stubs. +# Used for baseclasses that dont implement their own logging (Like ExecuteNode) +# Usually logging is implemented in subclasses (Like ZfsNode thats a subclass of ExecuteNode), but for regression testing its nice to have these stubs. class LogStub: """Just a stub, usually overriden in subclasses.""" @@ -15,4 +15,4 @@ class LogStub: print("WARNING: " + txt) def error(self, txt): - print("ERROR : " + txt) \ No newline at end of file + print("ERROR : " + txt) diff --git a/zfs_autobackup/Thinner.py b/zfs_autobackup/Thinner.py index 0718505..ec7a059 100644 --- a/zfs_autobackup/Thinner.py +++ b/zfs_autobackup/Thinner.py @@ -1,4 +1,3 @@ - from .ThinnerRule import ThinnerRule @@ -48,7 +47,6 @@ class Thinner: now: if specified, use this time as current time """ - # always keep a number of the last objets? if self.always_keep: # all of them @@ -70,24 +68,30 @@ class Thinner: # traverse objects for thisobject in objects: - # important they are ints! - timestamp = int(thisobject.timestamp) - age = int(now) - timestamp - # store in the correct time blocks, per period-size, if not too old yet - # e.g.: look if there is ANY timeblock that wants to keep this object - keep = False - for rule in self.rules: - if age <= rule.ttl: - block_nr = int(timestamp / rule.period) - if block_nr not in time_blocks[rule.period]: - time_blocks[rule.period][block_nr] = True - keep = True - - # keep it according to schedule, or keep it because it is in the keep_objects list - if keep or thisobject in keep_objects or thisobject in always_keep_objects: + # ignore stuff without timestamp, always keep those. + if thisobject.timestamp is None: keeps.append(thisobject) else: - removes.append(thisobject) - return keeps, removes \ No newline at end of file + # important they are ints! + timestamp = int(thisobject.timestamp) + age = int(now) - timestamp + + # store in the correct time blocks, per period-size, if not too old yet + # e.g.: look if there is ANY timeblock that wants to keep this object + keep = False + for rule in self.rules: + if age <= rule.ttl: + block_nr = int(timestamp / rule.period) + if block_nr not in time_blocks[rule.period]: + time_blocks[rule.period][block_nr] = True + keep = True + + # keep it according to schedule, or keep it because it is in the keep_objects list + if keep or thisobject in keep_objects or thisobject in always_keep_objects: + keeps.append(thisobject) + else: + removes.append(thisobject) + + return keeps, removes diff --git a/zfs_autobackup/ThinnerRule.py b/zfs_autobackup/ThinnerRule.py index 0bb1425..ab6f843 100644 --- a/zfs_autobackup/ThinnerRule.py +++ b/zfs_autobackup/ThinnerRule.py @@ -68,4 +68,4 @@ class ThinnerRule: def __str__(self): """get schedule as a schedule string""" - return self.rule_str \ No newline at end of file + return self.rule_str diff --git a/zfs_autobackup/TreeHasher.py b/zfs_autobackup/TreeHasher.py index b787475..0564b12 100644 --- a/zfs_autobackup/TreeHasher.py +++ b/zfs_autobackup/TreeHasher.py @@ -14,7 +14,7 @@ class TreeHasher(): :type block_hasher: BlockHasher """ - self.block_hasher=block_hasher + self.block_hasher = block_hasher def generate(self, start_path): """Use BlockHasher on every file in a tree, yielding the results @@ -28,12 +28,11 @@ class TreeHasher(): for (dirpath, dirnames, filenames) in os.walk(start_path, onerror=walkerror): for f in filenames: - file_path=os.path.join(dirpath, f) + file_path = os.path.join(dirpath, f) if (not os.path.islink(file_path)) and os.path.isfile(file_path): for (chunk_nr, hash) in self.block_hasher.generate(file_path): - yield ( os.path.relpath(file_path,start_path), chunk_nr, hash ) - + yield (os.path.relpath(file_path, start_path), chunk_nr, hash) def compare(self, start_path, generator): """reads from generator and compares blocks @@ -43,18 +42,14 @@ class TreeHasher(): """ - count=0 - - def filter_file_name( file_name, chunk_nr, hexdigest): - return ( chunk_nr, hexdigest ) + count = 0 + def filter_file_name(file_name, chunk_nr, hexdigest): + return (chunk_nr, hexdigest) for file_name, group_generator in itertools.groupby(generator, lambda x: x[0]): - count=count+1 - block_generator=itertools.starmap(filter_file_name, group_generator) - for ( chunk_nr, compare_hexdigest, actual_hexdigest) in self.block_hasher.compare(os.path.join(start_path,file_name), block_generator): - yield ( file_name, chunk_nr, compare_hexdigest, actual_hexdigest ) - - - - + count = count + 1 + block_generator = itertools.starmap(filter_file_name, group_generator) + for (chunk_nr, compare_hexdigest, actual_hexdigest) in self.block_hasher.compare( + os.path.join(start_path, file_name), block_generator): + yield (file_name, chunk_nr, compare_hexdigest, actual_hexdigest) diff --git a/zfs_autobackup/ZfsAuto.py b/zfs_autobackup/ZfsAuto.py index 4027fcb..b9b7035 100644 --- a/zfs_autobackup/ZfsAuto.py +++ b/zfs_autobackup/ZfsAuto.py @@ -36,81 +36,111 @@ class ZfsAuto(CliBase): self.warning("--ignore-replicated has been renamed, using --exclude-unchanged") args.exclude_unchanged = True + if args.exclude_received: + self.log.error("--exclude-received is no longer needed. zfs-autobackup now filters all the autobackup:... properties for newly received datasets to solve this problem. To get rid of existing properties on the target do this one time: 'zfs inherit -r autobackup:backup1 pool/backup1'") + sys.exit(255) + # Note: Before version v3.1-beta5, we always used exclude_received. This was a problem if you wanted to # replicate an existing backup to another host and use the same backupname/snapshots. However, exclude_received # may still need to be used to explicitly exclude a backup with the 'received' source property to avoid accidental # recursive replication of a zvol that is currently being received in another session (as it will have changes). + # Follow up note: This isnt a problem anymore since v4.0, we now filter the autobackup-property by default to prevent these difficult issues. + self.exclude_paths = [] if args.ssh_source == args.ssh_target: if args.target_path: # target and source are the same, make sure to exclude target_path self.verbose("NOTE: Source and target are on the same host, excluding target-path from selection.") self.exclude_paths.append(args.target_path) - else: - if not args.exclude_received and not args.include_received: - self.verbose("NOTE: Source and target are on the same host, adding --exclude-received to commandline. (use --include-received to overrule)") - args.exclude_received = True if args.test: self.warning("TEST MODE - SIMULATING WITHOUT MAKING ANY CHANGES") - #format all the names + # format all the names self.property_name = args.property_format.format(args.backup_name) self.snapshot_time_format = args.snapshot_format.format(args.backup_name) self.hold_name = args.hold_format.format(args.backup_name) + self.tag_seperator = args.tag_seperator dt = datetime_now(args.utc) self.verbose("") - self.verbose("Current time {} : {}".format(args.utc and "UTC" or " ", dt.strftime("%Y-%m-%d %H:%M:%S"))) + self.verbose( + "Current time {} : {}".format(args.utc and "UTC" or " ", dt.strftime("%Y-%m-%d %H:%M:%S"))) self.verbose("Selecting dataset property : {}".format(self.property_name)) self.verbose("Snapshot format : {}".format(self.snapshot_time_format)) self.verbose("Timezone : {}".format("UTC" if args.utc else "Local")) + seperator_test = datetime_now(False).strftime(self.snapshot_time_format) + + # according to man 8 zfs: + valid_tags = "_.: -" + if self.tag_seperator == '' or any(c not in valid_tags for c in self.tag_seperator): + self.log.error("Invalid tag seperator. Allowed characters: '{}'".format(valid_tags)) + sys.exit(255) + + if self.tag_seperator in seperator_test: + self.log.error("Tag seperator '{}' may not be used in snapshot format: {}".format(self.tag_seperator, + self.snapshot_time_format)) + sys.exit(255) + + if args.tag and self.tag_seperator in args.tag: + self.log.error( + "Tag '{}' may not contain tag seperator '{}'".format(args.tag, self.tag_seperator)) + sys.exit(255) + + if args.tag: + self.verbose("Tag : {}".format(self.tag_seperator + args.tag)) + return args def get_parser(self): parser = super(ZfsAuto, self).get_parser() - #positional arguments + # positional arguments parser.add_argument('backup_name', metavar='BACKUP-NAME', default=None, nargs='?', help='Name of the backup to select') parser.add_argument('target_path', metavar='TARGET-PATH', default=None, nargs='?', help='Target ZFS filesystem (optional)') - - # SSH options - group=parser.add_argument_group("SSH options") + group = parser.add_argument_group("SSH options") group.add_argument('--ssh-config', metavar='CONFIG-FILE', default=None, help='Custom ssh client config') group.add_argument('--ssh-source', metavar='USER@HOST', default=None, - help='Source host to pull backup from.') + help='Source host to pull backup from.') group.add_argument('--ssh-target', metavar='USER@HOST', default=None, - help='Target host to push backup to.') + help='Target host to push backup to.') - group=parser.add_argument_group("String formatting options") + group = parser.add_argument_group("String formatting options") group.add_argument('--property-format', metavar='FORMAT', default="autobackup:{}", - help='Dataset selection string format. Default: %(default)s') + help='Name of the ZFS user-property used to select datasets for this backup. ' + 'The literal "{}" is substituted with BACKUP-NAME. ' + 'Default: %(default)s') group.add_argument('--snapshot-format', metavar='FORMAT', default="{}-%Y%m%d%H%M%S", - help='ZFS Snapshot string format. Default: %(default)s') + help='Format of the snapshot-name part after the "@". ' + 'The literal "{}" is substituted with BACKUP-NAME, then strftime-codes ' + '(e.g. %%Y %%m %%d %%H %%M %%S) are expanded against the current time. ' + 'Must not contain the tag-seperator. Default: %(default)s') group.add_argument('--hold-format', metavar='FORMAT', default="zfs_autobackup:{}", - help='ZFS hold string format. Default: %(default)s') + help='Name of the zfs-hold placed on snapshots to prevent accidental deletion. ' + 'The literal "{}" is substituted with BACKUP-NAME. ' + 'Default: %(default)s') group.add_argument('--strip-path', metavar='N', default=0, type=int, help='Number of directories to strip from target path.') + group.add_argument('--tag-seperator', metavar='STRING', default="__", + help="Tag seperator for snapshots and bookmarks. Default: %(default)s") + group.add_argument('--tag', metavar='TAG', default=None, + help='Backup tag to add to snapshots names. (For administrative purposes)') - group=parser.add_argument_group("Selection options") + group = parser.add_argument_group("Selection options") group.add_argument('--ignore-replicated', action='store_true', help=argparse.SUPPRESS) group.add_argument('--exclude-unchanged', metavar='BYTES', default=0, type=int, - help='Exclude datasets that have less than BYTES data changed since any last snapshot. (Use with proxmox HA replication)') - group.add_argument('--exclude-received', action='store_true', - help='Exclude datasets that have the origin of their autobackup: property as "received". ' - 'This can avoid recursive replication between two backup partners.') - group.add_argument('--include-received', action='store_true', - help=argparse.SUPPRESS) + help='Exclude datasets that have less than BYTES data changed since any last snapshot. (Use with proxmox HA replication)') + group.add_argument('--exclude-received', action='store_true', help=argparse.SUPPRESS) def regex_argument_type(input_line): @@ -119,16 +149,14 @@ class ZfsAuto(CliBase): return re.compile(input_line) except: raise ValueError("Could not parse argument '{}' as a regular expression".format(input_line)) - group.add_argument('--exclude-snapshot-pattern', action='append', default=[], type=regex_argument_type, help="Regular expression to match snapshots that will be ignored.") + + group.add_argument('--exclude-snapshot-pattern', action='append', default=[], type=regex_argument_type, + help="Regular expression to match snapshots that will be ignored.") return parser def print_error_sources(self): self.error( - "No source filesystems selected, please do a 'zfs set autobackup:{0}=true' on the source datasets " - "you want to select.".format( - self.args.backup_name)) + "No source filesystems selected, please do a 'zfs set {}=true' on the source datasets " + "you want to select.".format(self.property_name)) - def make_target_name(self, source_dataset): - """make target_name from a source_dataset""" - return self.args.target_path + "/" + source_dataset.lstrip_path(self.args.strip_path) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 655bb2e..bc483a9 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -1,16 +1,14 @@ - import argparse from signal import signal, SIGPIPE -from .util import output_redir, sigpipe_handler, datetime_now - -from .ZfsAuto import ZfsAuto from . import compressors from .ExecuteNode import ExecuteNode from .Thinner import Thinner -from .ZfsDataset import ZfsDataset +from .ZfsAuto import ZfsAuto from .ZfsNode import ZfsNode -from .ThinnerRule import ThinnerRule +from .sync_datasets import sync_datasets +from .util import sigpipe_handler, datetime_now + class ZfsAutobackup(ZfsAuto): """The main zfs-autobackup class. Start here, at run() :)""" @@ -31,6 +29,15 @@ class ZfsAutobackup(ZfsAuto): if args.allow_empty: args.min_change = 0 + # With --keep-source=0 nothing survives on the source between runs, so the + # per-dataset is_changed_ours() check has no reference snapshot to compare + # against — every dataset would get a fresh snapshot anyway. Force min-change=0 + # to skip the our_snapshots / written@... lookups entirely (significant speedup + # on large dataset counts). + if args.keep_source == "0" and args.min_change > 0: + self.verbose("NOTE: --keep-source=0: forcing --min-change=0 (no source reference to compare against).") + args.min_change = 0 + # if args.destroy_incompatible: # args.rollback = True @@ -47,6 +54,10 @@ class ZfsAutobackup(ZfsAuto): if args.compress and args.zfs_compressed: self.warning("Using --compress with --zfs-compressed, might be inefficient.") + if args.decrypt: + self.warning( + "Properties will not be sent over for datasets that will be decrypted. (zfs bug https://github.com/openzfs/zfs/issues/16275)") + return args def get_parser(self): @@ -73,12 +84,17 @@ class ZfsAutobackup(ZfsAuto): group.add_argument('--no-guid-check', action='store_true', help='Dont check guid of common snapshots. (faster)') - group = parser.add_argument_group("Transfer options") group.add_argument('--no-send', action='store_true', help='Don\'t transfer snapshots (useful for cleanups, or if you want a separate send-cronjob)') group.add_argument('--no-holds', action='store_true', help='Don\'t hold snapshots. (Faster. Allows you to destroy common snapshot.)') + group.add_argument('--no-bookmarks', action='store_true', + help='Don\'t use bookmarks.') + group.add_argument('--no-clone', action='store_true', + help="Don't preserve clone relationships during replication. " + "Source datasets that are clones will be replicated as full standalone datasets.") + group.add_argument('--clear-refreservation', action='store_true', help='Filter "refreservation" property. (recommended, saves space. same as ' '--filter-properties refreservation)') @@ -91,6 +107,7 @@ class ZfsAutobackup(ZfsAuto): group.add_argument('--set-properties', metavar='PROPERTY=VALUE,...', type=str, help='List of propererties to override when receiving filesystems. (you can still restore ' 'them with zfs inherit -S)') + group.add_argument('--rollback', action='store_true', help='Rollback changes to the latest target snapshot before starting. (normally you can ' 'prevent changes by setting the readonly property on the target_path to on)') @@ -110,12 +127,6 @@ class ZfsAutobackup(ZfsAuto): group.add_argument('--zfs-compressed', action='store_true', help='Transfer blocks that already have zfs-compression as-is.') - group.add_argument('--clones', metavar='POLICY', default='never', - choices=('never', 'simple'), - help='Clones support. (The default policy "never" expands clones into full independent datasets. ' - '"simple" tries to reproduce the clone when the origin snapshot is already copied ' - 'in the same target root)') - group = parser.add_argument_group("Data transfer options") group.add_argument('--compress', metavar='TYPE', default=None, nargs='?', const='zstd-fast', choices=compressors.choices(), @@ -125,8 +136,8 @@ class ZfsAutobackup(ZfsAuto): help='Limit data transfer rate in Bytes/sec (e.g. 128K. requires mbuffer.)') group.add_argument('--buffer', metavar='SIZE', default=None, help='Add zfs send and recv buffers to smooth out IO bursts. (e.g. 128M. requires mbuffer)') - parser.add_argument('--buffer-chunk-size', metavar="BUFFERCHUNKSIZE", default=None, - help='Tune chunk size when mbuffer is used. (requires mbuffer.)') + group.add_argument('--buffer-chunk-size', metavar="BUFFERCHUNKSIZE", default=None, + help='Tune chunk size when mbuffer is used. (requires mbuffer.)') group.add_argument('--send-pipe', metavar="COMMAND", default=[], action='append', help='pipe zfs send output through COMMAND (can be used multiple times)') group.add_argument('--recv-pipe', metavar="COMMAND", default=[], action='append', @@ -148,104 +159,6 @@ class ZfsAutobackup(ZfsAuto): return parser - # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: - def thin_missing_targets(self, target_dataset, used_target_datasets): - """thin target datasets that are missing on the source. - :type used_target_datasets: list[ZfsDataset] - :type target_dataset: ZfsDataset - """ - - self.debug("Thinning obsolete datasets") - missing_datasets = [dataset for dataset in target_dataset.recursive_datasets if - dataset not in used_target_datasets] - - count = 0 - for dataset in missing_datasets: - self.debug("analyse missing {}".format(dataset)) - - count = count + 1 - if self.args.progress: - self.progress("Analysing missing {}/{}".format(count, len(missing_datasets))) - - try: - dataset.debug("Missing on source, thinning") - dataset.thin() - - except Exception as e: - dataset.error("Error during thinning of missing datasets ({})".format(str(e))) - - # if self.args.progress: - # self.clear_progress() - - # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: - def destroy_missing_targets(self, target_dataset, used_target_datasets): - """destroy target datasets that are missing on the source and that meet the requirements - :type used_target_datasets: list[ZfsDataset] - :type target_dataset: ZfsDataset - - """ - - self.debug("Destroying obsolete datasets") - - missing_datasets = [dataset for dataset in target_dataset.recursive_datasets if - dataset not in used_target_datasets] - - count = 0 - for dataset in missing_datasets: - - count = count + 1 - if self.args.progress: - self.progress("Analysing destroy missing {}/{}".format(count, len(missing_datasets))) - - try: - # cant do anything without our own snapshots - if not dataset.our_snapshots: - if dataset.datasets: - # its not a leaf, just ignore - dataset.debug("Destroy missing: ignoring") - else: - dataset.verbose( - "Destroy missing: has no snapshots made by us (please destroy manually).") - else: - # past the deadline? - deadline_ttl = ThinnerRule("0s" + self.args.destroy_missing).ttl - now = datetime_now(self.args.utc).timestamp() - if dataset.our_snapshots[-1].timestamp + deadline_ttl > now: - dataset.verbose("Destroy missing: Waiting for deadline.") - else: - - dataset.debug("Destroy missing: Removing our snapshots.") - - # remove all our snaphots, except last, to safe space in case we fail later on - for snapshot in dataset.our_snapshots[:-1]: - snapshot.destroy(fail_exception=True) - - # does it have other snapshots? - has_others = False - for snapshot in dataset.snapshots: - if not snapshot.is_ours(): - has_others = True - break - - if has_others: - dataset.verbose("Destroy missing: Still in use by other snapshots") - else: - if dataset.datasets: - dataset.verbose("Destroy missing: Still has children here.") - else: - dataset.verbose("Destroy missing.") - dataset.our_snapshots[-1].destroy(fail_exception=True) - dataset.destroy(fail_exception=True) - - except Exception as e: - # if self.args.progress: - # self.clear_progress() - - dataset.error("Error during --destroy-missing: {}".format(str(e))) - - # if self.args.progress: - # self.clear_progress() - def get_send_pipes(self, logger): """determine the zfs send pipe""" @@ -328,102 +241,24 @@ class ZfsAutobackup(ZfsAuto): return ret - def make_target_name(self, source_dataset): - """make target_name from a source_dataset""" - stripped=source_dataset.lstrip_path(self.args.strip_path) - if stripped!="": - return self.args.target_path + "/" + stripped - else: - return self.args.target_path - def check_target_names(self, source_node, source_datasets, target_node): """check all target names for collesions etc due to strip-options""" self.debug("Checking target names:") - target_datasets={} + target_datasets = {} for source_dataset in source_datasets: - target_name = self.make_target_name(source_dataset) + target_name = source_dataset.map_to_target_path(self.args.target_path, self.args.strip_path) source_dataset.debug("-> {}".format(target_name)) if target_name in target_datasets: - raise Exception("Target collision: Target path {} encountered twice, due to: {} and {}".format(target_name, source_dataset, target_datasets[target_name])) + raise Exception( + "Target collision: Target path {} encountered twice, due to: {} and {}".format(target_name, + source_dataset, + target_datasets[ + target_name])) - target_datasets[target_name]=source_dataset - - # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: - def sync_datasets(self, source_node, source_datasets, target_node): - """Sync datasets, or thin-only on both sides - :type target_node: ZfsNode - :type source_datasets: list of ZfsDataset - :type source_node: ZfsNode - """ - - send_pipes = self.get_send_pipes(source_node.verbose) - recv_pipes = self.get_recv_pipes(target_node.verbose) - - fail_count = 0 - count = 0 - target_datasets = [] - for source_dataset in source_datasets: - - # stats - if self.args.progress: - count = count + 1 - self.progress("Analysing dataset {}/{} ({} failed)".format(count, len(source_datasets), fail_count)) - - try: - # determine corresponding target_dataset - target_name = self.make_target_name(source_dataset) - target_dataset = target_node.get_dataset(target_name) - target_datasets.append(target_dataset) - - # ensure parents exists - # TODO: this isnt perfect yet, in some cases it can create parents when it shouldn't. - if not self.args.no_send \ - and target_dataset.parent \ - and target_dataset.parent not in target_datasets \ - and not target_dataset.parent.exists: - target_dataset.debug("Creating unmountable parents") - target_dataset.parent.create_filesystem(parents=True) - - # determine common zpool features (cached, so no problem we call it often) - source_features = source_node.get_pool(source_dataset).features - target_features = target_node.get_pool(target_dataset).features - common_features = source_features and target_features - - # sync the snapshots of this dataset - source_dataset.sync_snapshots(target_dataset, show_progress=self.args.progress, - features=common_features, filter_properties=self.filter_properties_list(), - set_properties=self.set_properties_list(), - ignore_recv_exit_code=self.args.ignore_transfer_errors, - holds=not self.args.no_holds, rollback=self.args.rollback, - also_other_snapshots=self.args.other_snapshots, - no_send=self.args.no_send, - destroy_incompatible=self.args.destroy_incompatible, - send_pipes=send_pipes, recv_pipes=recv_pipes, - decrypt=self.args.decrypt, encrypt=self.args.encrypt, - zfs_compressed=self.args.zfs_compressed, force=self.args.force, - guid_check=not self.args.no_guid_check, - clones=self.args.clones, - make_target_name=lambda source_dataset: self.make_target_name(source_dataset)) - except Exception as e: - - fail_count = fail_count + 1 - source_dataset.error("FAILED: " + str(e)) - if self.args.debug: - self.verbose("Debug mode, aborting on first error") - raise - - - target_path_dataset = target_node.get_dataset(self.args.target_path) - if not self.args.no_thinning: - self.thin_missing_targets(target_dataset=target_path_dataset, used_target_datasets=target_datasets) - - if self.args.destroy_missing is not None: - self.destroy_missing_targets(target_dataset=target_path_dataset, used_target_datasets=target_datasets) - - return fail_count + target_datasets[target_name] = source_dataset def thin_source(self, source_datasets): @@ -482,14 +317,17 @@ class ZfsAutobackup(ZfsAuto): ssh_config=self.args.ssh_config, ssh_to=self.args.ssh_source, readonly=self.args.test, debug_output=self.args.debug_output, description=description, thinner=source_thinner, - exclude_snapshot_patterns=self.args.exclude_snapshot_pattern) + exclude_snapshot_patterns=self.args.exclude_snapshot_pattern, + tag_seperator=self.tag_seperator) ################# select source datasets self.set_title("Selecting") - ( source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, - exclude_received=self.args.exclude_received, - exclude_paths=self.exclude_paths, - exclude_unchanged=self.args.exclude_unchanged) + if self.args.progress: + self.progress(f"Loading...") + + (source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, + exclude_paths=self.exclude_paths, + exclude_unchanged=self.args.exclude_unchanged) if not source_datasets and not excluded_datasets: self.print_error_sources() return 255 @@ -498,6 +336,9 @@ class ZfsAutobackup(ZfsAuto): if not self.args.no_snapshot: self.set_title("Snapshotting") snapshot_name = datetime_now(self.args.utc).strftime(self.snapshot_time_format) + if self.args.tag: + snapshot_name = snapshot_name + self.tag_seperator + self.args.tag + source_node.consistent_snapshot(source_datasets, snapshot_name, min_changed_bytes=self.args.min_change, pre_snapshot_cmds=self.args.pre_snapshot_cmd, @@ -520,26 +361,57 @@ class ZfsAutobackup(ZfsAuto): ssh_to=self.args.ssh_target, readonly=self.args.test, debug_output=self.args.debug_output, description="[Target]", - thinner=target_thinner) + exclude_snapshot_patterns=self.args.exclude_snapshot_pattern, + thinner=target_thinner, tag_seperator=self.tag_seperator) target_node.verbose("Receive datasets under: {}".format(self.args.target_path)) - self.set_title("Synchronising") - # check if exists, to prevent vague errors - target_dataset = target_node.get_dataset(self.args.target_path) + target_dataset = target_node.get_container(self.args.target_path) if not target_dataset.exists: raise (Exception( "Target path '{}' does not exist. Please create this dataset first.".format(target_dataset))) + bookmark_tag = target_dataset.properties['guid'] + target_node.verbose("Bookmark tag: {}".format(bookmark_tag)) + + self.set_title("Synchronising") + # check for collisions due to strip-path self.check_target_names(source_node, source_datasets, target_node) # do the actual sync # NOTE: even with no_send, no_thinning and no_snapshot it does a usefull thing because it checks if the common snapshots and shows incompatible snapshots - fail_count = self.sync_datasets( + fail_count = sync_datasets( + self.log, source_node=source_node, source_datasets=source_datasets, - target_node=target_node) + target_node=target_node, + bookmark_tag=bookmark_tag, + send_pipes=self.get_send_pipes(source_node.verbose), + recv_pipes=self.get_recv_pipes(target_node.verbose), + target_path=self.args.target_path, + strip_path=self.args.strip_path, + no_clone=self.args.no_clone, + no_send=self.args.no_send, + no_bookmarks=self.args.no_bookmarks, + no_thinning=self.args.no_thinning, + filter_properties=self.filter_properties_list(), + set_properties=self.set_properties_list(), + ignore_transfer_errors=self.args.ignore_transfer_errors, + holds=not self.args.no_holds, + rollback=self.args.rollback, + other_snapshots=self.args.other_snapshots, + destroy_incompatible=self.args.destroy_incompatible, + decrypt=self.args.decrypt, + encrypt=self.args.encrypt, + zfs_compressed=self.args.zfs_compressed, + force=self.args.force, + guid_check=not self.args.no_guid_check, + property_format=self.args.property_format, + debug=self.args.debug, + target_dataset_base=target_dataset, + destroy_missing=self.args.destroy_missing, + utc=self.args.utc) # no target specified, run in snapshot-only mode else: @@ -581,7 +453,7 @@ def cli(): signal(SIGPIPE, sigpipe_handler) - failed_datasets=ZfsAutobackup(sys.argv[1:], False).run() + failed_datasets = ZfsAutobackup(sys.argv[1:], False).run() sys.exit(min(failed_datasets, 255)) diff --git a/zfs_autobackup/ZfsAutoverify.py b/zfs_autobackup/ZfsAutoverify.py index 96abd94..0f75732 100644 --- a/zfs_autobackup/ZfsAutoverify.py +++ b/zfs_autobackup/ZfsAutoverify.py @@ -76,14 +76,14 @@ def verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt, source_snapshot.mount(source_mnt) target_snapshot.mount(target_mnt) - if method=='rsync': + if method == 'rsync': compare_trees_rsync(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt) # elif method == 'tar': # compare_trees_tar(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt) elif method == 'find': compare_trees_find(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt) else: - raise(Exception("program errror, unknown method")) + raise (Exception("program errror, unknown method")) finally: source_snapshot.unmount(source_mnt) @@ -109,7 +109,6 @@ def verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt, # return hashed - # def deacitvate_volume_snapshot(snapshot): # clone_name=get_tmp_clone_name(snapshot) # clone=snapshot.zfs_node.get_dataset(clone_name) @@ -119,13 +118,13 @@ def verify_volume(source_dataset, source_snapshot, target_dataset, target_snapsh """compare the contents of two zfs volume snapshots""" # try: - source_dev= activate_volume_snapshot(source_snapshot) - target_dev= activate_volume_snapshot(target_snapshot) + source_dev = activate_volume_snapshot(source_snapshot) + target_dev = activate_volume_snapshot(target_snapshot) - source_hash= hash_dev(source_snapshot.zfs_node, source_dev) - target_hash= hash_dev(target_snapshot.zfs_node, target_dev) + source_hash = hash_dev(source_snapshot.zfs_node, source_dev) + target_hash = hash_dev(target_snapshot.zfs_node, target_dev) - if source_hash!=target_hash: + if source_hash != target_hash: raise Exception("md5hash difference: {} != {}".format(source_hash, target_hash)) # finally: @@ -150,7 +149,7 @@ class ZfsAutoverify(ZfsAuto): def parse_args(self, argv): """do extra checks on common args""" - args=super(ZfsAutoverify, self).parse_args(argv) + args = super(ZfsAutoverify, self).parse_args(argv) if args.target_path == None: self.log.error("Please specify TARGET-PATH") @@ -161,17 +160,17 @@ class ZfsAutoverify(ZfsAuto): def get_parser(self): """extend common parser with extra stuff needed for zfs-autobackup""" - parser=super(ZfsAutoverify, self).get_parser() + parser = super(ZfsAutoverify, self).get_parser() - group=parser.add_argument_group("Verify options") + group = parser.add_argument_group("Verify options") group.add_argument('--fs-compare', metavar='METHOD', default="find", choices=["find", "rsync"], - help='Compare method to use for filesystems. (find, rsync) Default: %(default)s ') + help='Compare method to use for filesystems. (find, rsync) Default: %(default)s ') return parser def verify_datasets(self, source_mnt, source_datasets, target_node, target_mnt): - fail_count=0 + fail_count = 0 count = 0 for source_dataset in source_datasets: @@ -182,24 +181,24 @@ class ZfsAutoverify(ZfsAuto): try: # determine corresponding target_dataset - target_name = self.make_target_name(source_dataset) - target_dataset = target_node.get_dataset(target_name) + target_dataset = target_node.get_container(source_dataset.map_to_target_path(self.args.target_path, self.args.strip_path)) # find common snapshots to verify source_snapshot = source_dataset.find_common_snapshot(target_dataset, True) target_snapshot = target_dataset.find_snapshot(source_snapshot) if source_snapshot is None or target_snapshot is None: - raise(Exception("Cant find common snapshot")) + raise (Exception("Cant find common snapshot")) target_snapshot.verbose("Verifying...") - if source_dataset.properties['type']=="filesystem": + if source_dataset.properties['type'] == "filesystem": verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt, self.args.fs_compare) - elif source_dataset.properties['type']=="volume": + elif source_dataset.properties['type'] == "volume": verify_volume(source_dataset, source_snapshot, target_dataset, target_snapshot) else: - raise(Exception("{} has unknown type {}".format(source_dataset, source_dataset.properties['type']))) + raise ( + Exception("{} has unknown type {}".format(source_dataset, source_dataset.properties['type']))) except Exception as e: @@ -219,11 +218,10 @@ class ZfsAutoverify(ZfsAuto): def run(self): - source_node=None - source_mnt=None - target_node=None - target_mnt=None - + source_node = None + source_mnt = None + target_node = None + target_mnt = None try: @@ -240,10 +238,9 @@ class ZfsAutoverify(ZfsAuto): ################# select source datasets self.set_title("Selecting") - ( source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, - exclude_received=self.args.exclude_received, - exclude_paths=self.exclude_paths, - exclude_unchanged=self.args.exclude_unchanged) + (source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, + exclude_paths=self.exclude_paths, + exclude_unchanged=self.args.exclude_unchanged) if not source_datasets and not excluded_datasets: self.print_error_sources() return 255 @@ -260,7 +257,7 @@ class ZfsAutoverify(ZfsAuto): self.set_title("Verifying") - source_mnt, target_mnt= create_mountpoints(source_node, target_node) + source_mnt, target_mnt = create_mountpoints(source_node, target_node) fail_count = self.verify_datasets( source_mnt=source_mnt, @@ -302,15 +299,13 @@ class ZfsAutoverify(ZfsAuto): cleanup_mountpoint(target_node, target_mnt) - - def cli(): import sys - raise(Exception("This program is incomplete, dont use it yet.")) + raise (Exception("This program is incomplete, dont use it yet.")) signal(SIGPIPE, sigpipe_handler) failed = ZfsAutoverify(sys.argv[1:], False).run() - sys.exit(min(failed,255)) + sys.exit(min(failed, 255)) if __name__ == "__main__": diff --git a/zfs_autobackup/ZfsBookmark.py b/zfs_autobackup/ZfsBookmark.py new file mode 100644 index 0000000..418bab0 --- /dev/null +++ b/zfs_autobackup/ZfsBookmark.py @@ -0,0 +1,26 @@ +from .ZfsPointInTime import ZfsPointInTime + + +class ZfsBookmark(ZfsPointInTime): + """A ZFS bookmark""" + + @property + def prefix(self): + (filesystem, snapshot) = self.name.split("#") + return filesystem + + @property + def suffix(self): + (filesystem, snapshot_name) = self.name.split("#") + return snapshot_name + + @property + def typed_suffix(self): + return "#" + self.suffix + + + + def destroy(self, fail_exception=False, **kwargs): + + self.debug("Destroying") + return super().destroy(fail_exception=fail_exception) diff --git a/zfs_autobackup/ZfsCheck.py b/zfs_autobackup/ZfsCheck.py index 10fe8ae..dab17a1 100644 --- a/zfs_autobackup/ZfsCheck.py +++ b/zfs_autobackup/ZfsCheck.py @@ -9,8 +9,10 @@ from .BlockHasher import BlockHasher from .ZfsNode import ZfsNode from .util import * from .CliBase import CliBase +from .ZfsSnapshot import ZfsSnapshot +# NOTE: zfscheck has no ssh support, because it would be too problematic during cleanup of temporary mounts and clones. class ZfsCheck(CliBase): def __init__(self, argv, print_arguments=True): @@ -18,7 +20,9 @@ class ZfsCheck(CliBase): # NOTE: common options argument parsing are in CliBase super(ZfsCheck, self).__init__(argv, print_arguments) - self.node = ZfsNode(self.log, utc=self.args.utc, readonly=self.args.test, debug_output=self.args.debug_output) + self.node = ZfsNode(self.log, utc=self.args.utc, readonly=self.args.test, debug_output=self.args.debug_output, + ssh_to=None, ssh_config=None, exclude_snapshot_patterns=None, thinner=None, + tag_seperator='__', description="", hold_name=None, snapshot_time_format=None) self.block_hasher = BlockHasher(count=self.args.count, bs=self.args.block_size, skip=self.args.skip) @@ -27,14 +31,16 @@ class ZfsCheck(CliBase): parser = super(ZfsCheck, self).get_parser() # positional arguments - parser.add_argument('target', metavar='TARGET', default=None, nargs='?', help='Target to checkum. (can be blockdevice, directory or ZFS snapshot)') + parser.add_argument('target', metavar='TARGET', default=None, nargs='?', + help='Target to checkum. (can be blockdevice, directory or ZFS snapshot)') group = parser.add_argument_group('Checker options') group.add_argument('--block-size', metavar="BYTES", default=4096, help="Read block-size, default %(default)s", type=int) group.add_argument('--count', metavar="COUNT", default=int((100 * (1024 ** 2)) / 4096), - help="Hash chunks of COUNT blocks. Default %(default)s . (CHUNK size is BYTES * COUNT) ", type=int) # 100MiB + help="Hash chunks of COUNT blocks. Default %(default)s . (CHUNK size is BYTES * COUNT) ", + type=int) # 100MiB group.add_argument('--check', '-c', metavar="FILE", default=None, const=True, nargs='?', help="Read hashes from STDIN (or FILE) and compare them") @@ -57,15 +63,17 @@ class ZfsCheck(CliBase): self.verbose("Target : {}".format(args.target)) self.verbose("Block size : {} bytes".format(args.block_size)) self.verbose("Block count : {}".format(args.count)) - self.verbose("Effective chunk size : {} bytes".format(args.count*args.block_size)) - self.verbose("Skip chunk count : {} (checks {:.2f}% of data)".format(args.skip, 100/(1+args.skip))) + self.verbose("Effective chunk size : {} bytes".format(args.count * args.block_size)) + self.verbose("Skip chunk count : {} (checks {:.2f}% of data)".format(args.skip, 100 / (1 + args.skip))) self.verbose("") - return args def prepare_zfs_filesystem(self, snapshot): - + """ + :type snapshot: ZfsSnapshot + :rtype: str + """ mnt = "/tmp/" + tmp_name() self.debug("Create temporary mount point {}".format(mnt)) self.node.run(["mkdir", mnt]) @@ -73,6 +81,9 @@ class ZfsCheck(CliBase): return mnt def cleanup_zfs_filesystem(self, snapshot): + """ + :type snapshot: ZfsSnapshot + """ mnt = "/tmp/" + tmp_name() snapshot.unmount(mnt) self.debug("Cleaning up temporary mount point") @@ -81,7 +92,10 @@ class ZfsCheck(CliBase): # NOTE: https://www.google.com/search?q=Mount+Path+Limit+freebsd # Freebsd has limitations regarding path length, so we have to clone it so the part stays sort def prepare_zfs_volume(self, snapshot): - """clone volume, waits and tries to findout /dev path to the volume, in a compatible way. (linux/freebsd/smartos)""" + """clone volume, waits and tries to findout /dev path to the volume, in a compatible way. (linux/freebsd/smartos) + :type snapshot: ZfsSnapshot + :rtype: str + """ clone_name = get_tmp_clone_name(snapshot) clone = snapshot.clone(clone_name) @@ -106,13 +120,18 @@ class ZfsCheck(CliBase): time.sleep(1) - raise (Exception("Timeout while waiting for /dev entry to appear. (looking in: {}). Hint: did you forget to load the encryption key?".format(locations))) + raise (Exception( + "Timeout while waiting for /dev entry to appear. (looking in: {}). Hint: did you forget to load the encryption key?".format( + locations))) def cleanup_zfs_volume(self, snapshot): - """destroys temporary volume snapshot""" + """destroys temporary volume snapshot + :type snapshot: ZfsSnapshot + + """ clone_name = get_tmp_clone_name(snapshot) - clone = snapshot.zfs_node.get_dataset(clone_name) - clone.destroy(deferred=True, verbose=False) + clone = snapshot.zfs_node.get_container(clone_name) + clone.destroy() def generate_tree_hashes(self, prepared_target): @@ -144,34 +163,34 @@ class ZfsCheck(CliBase): """parse input lines and yield items to use in compare functions""" if self.args.check is True: - input_fh=sys.stdin + input_fh = sys.stdin else: - input_fh=open(self.args.check, 'r') + input_fh = open(self.args.check, 'r') last_progress_time = time.time() progress_checked = 0 progress_skipped = 0 - line=input_fh.readline() - skip=0 + line = input_fh.readline() + skip = 0 while line: - i=line.rstrip().split("\t") - #ignores lines without tabs - if (len(i)>1): + i = line.rstrip().split("\t") + # ignores lines without tabs + if (len(i) > 1): - if skip==0: - progress_checked=progress_checked+1 + if skip == 0: + progress_checked = progress_checked + 1 yield i - skip=self.args.skip + skip = self.args.skip else: - skip=skip-1 - progress_skipped=progress_skipped+1 + skip = skip - 1 + progress_skipped = progress_skipped + 1 if self.args.progress and time.time() - last_progress_time > 1: last_progress_time = time.time() self.progress("Checked {} hashes (skipped {})".format(progress_checked, progress_skipped)) - line=input_fh.readline() + line = input_fh.readline() self.verbose("Checked {} hashes (skipped {})".format(progress_checked, progress_skipped)) @@ -224,7 +243,7 @@ class ZfsCheck(CliBase): if "@" in self.args.target: # zfs snapshot - snapshot=self.node.get_dataset(self.args.target) + snapshot = self.node.get_snapshot(self.args.target) if not snapshot.exists: raise Exception("ZFS snapshot {} does not exist!".format(snapshot)) dataset_type = snapshot.parent.properties['type'] @@ -240,7 +259,7 @@ class ZfsCheck(CliBase): def cleanup_target(self): if "@" in self.args.target: # zfs snapshot - snapshot=self.node.get_dataset(self.args.target) + snapshot = self.node.get_snapshot(self.args.target) if not snapshot.exists: return @@ -253,28 +272,28 @@ class ZfsCheck(CliBase): def run(self): - compare_generator=None - hash_generator=None + compare_generator = None + hash_generator = None try: - prepared_target=self.prepare_target() - is_dir=os.path.isdir(prepared_target) + prepared_target = self.prepare_target() + is_dir = os.path.isdir(prepared_target) - #run as compare + # run as compare if self.args.check is not None: - input_generator=self.generate_input() + input_generator = self.generate_input() if is_dir: compare_generator = self.generate_tree_compare(prepared_target, input_generator) else: - compare_generator=self.generate_file_compare(prepared_target, input_generator) - errors=self.print_errors(compare_generator) - #run as generator + compare_generator = self.generate_file_compare(prepared_target, input_generator) + errors = self.print_errors(compare_generator) + # run as generator else: if is_dir: hash_generator = self.generate_tree_hashes(prepared_target) else: - hash_generator=self.generate_file_hashes(prepared_target) + hash_generator = self.generate_file_hashes(prepared_target) - errors=self.print_hashes(hash_generator) + errors = self.print_hashes(hash_generator) except Exception as e: self.error("Exception: " + str(e)) @@ -286,10 +305,10 @@ class ZfsCheck(CliBase): return 255 finally: - #important to call check_output so that cleanup still functions in case of a broken pipe: + # important to call check_output so that cleanup still functions in case of a broken pipe: # util.check_output() - #close generators, to make sure files are not in use anymore when cleaning up + # close generators, to make sure files are not in use anymore when cleaning up if hash_generator is not None: hash_generator.close() if compare_generator is not None: @@ -302,8 +321,8 @@ class ZfsCheck(CliBase): def cli(): import sys signal(SIGPIPE, sigpipe_handler) - failed=ZfsCheck(sys.argv[1:], False).run() - sys.exit(min(failed,255)) + failed = ZfsCheck(sys.argv[1:], False).run() + sys.exit(min(failed, 255)) if __name__ == "__main__": diff --git a/zfs_autobackup/ZfsContainer.py b/zfs_autobackup/ZfsContainer.py new file mode 100644 index 0000000..2cde684 --- /dev/null +++ b/zfs_autobackup/ZfsContainer.py @@ -0,0 +1,622 @@ +from typing import cast + +from .ZfsBookmark import ZfsBookmark +from .ZfsDataset import ZfsDataset +from .ZfsPointInTime import ZfsPointInTime +from .ZfsSnapshot import ZfsSnapshot + + + + + +class ZfsContainer(ZfsDataset): + """Either a ZFS Filesystem or ZFS Dataset""" + + def __init__(self, zfs_node, name, force_exists=None): + + super().__init__(zfs_node, name, force_exists=force_exists) + + self._written_since_ours = None # type: None|int + self._recursive_datasets = None # type: None|list[ZfsContainer] + self._datasets = None # type: None|list[ZfsContainer] + self._snapshots_bookmarks = None # type: None|list[ZfsSnapshot|ZfsBookmark] + + + # def __str__(self): + # + # return f"{self.zfs_node}: {self.name} {'force_exists' if self.force_exists else ''} {len(self._snapshots_bookmarks) if self._snapshots_bookmarks is not None else ''}" + + + + @property + def parent(self): + """get parent dataset + + :rtype: ZfsContainer | None + """ + stripped = self.rstrip_path(1) + if stripped: + return self.zfs_node.get_container(stripped) + else: + return None + + def map_to_target_path(self, target_path, strip_path): + """Map this dataset's name to a target path by stripping strip_path components and prepending target_path. + :type target_path: str + :type strip_path: int + :rtype: str + """ + stripped = self.lstrip_path(strip_path) + if stripped: + return target_path + "/" + stripped + else: + return target_path + + def invalidate_cache(self): + super().invalidate_cache() + self._written_since_ours = None + self._recursive_datasets = None + self._datasets = None + self._snapshots_bookmarks = None + + @property + def snapshots(self): + """get all snapshots of this dataset + :rtype: list[ZfsSnapshot] + """ + ret = [] + + for snapshot in self.snapshots_bookmarks: + # check is snapshot is a ZfsSnapshot, and append to ret if it is + if isinstance(snapshot, ZfsSnapshot): + ret.append(snapshot) + + return ret + + @property + def bookmarks(self): + """get all bookmarks of this dataset + Args: + + :rtype: list[ZfsBookmark] + """ + + ret = [] + + for bookmark in self.snapshots_bookmarks: + if type(bookmark) is ZfsBookmark: + ret.append(bookmark) + + return ret + + @property + def snapshots_bookmarks(self): + """get all snapshots and bookmarks of this dataset (ordered by createtxg, so its suitable to determine incremental zfs send order) + :rtype: list[ZfsSnapshot|ZfsBookmark] + """ + + # cached? + if self._snapshots_bookmarks is None: + self.debug("Getting snapshots and bookmarks") + + cmd = [ + "zfs", "list", "-d", "1", "-r", "-t", "snapshot,bookmark", "-H", "-o", "name", "-s", "createtxg", + self.name + ] + + names = cast("list[str]", self.zfs_node.run(cmd=cmd, readonly=True)) + self._snapshots_bookmarks = cast("list[ZfsSnapshot|ZfsBookmark]", + self.zfs_node.get_datasets(names, force_exists=True)) + + return self._snapshots_bookmarks + + def find_incompatible_snapshots(self, target_common_snapshot, raw): + """returns a list[snapshots] that is incompatible for a zfs recv onto + the common_snapshot. all direct followup snapshots with written=0 are + compatible. + + in raw-mode nothing is compatible. issue #219 + + Args: + :type target_common_snapshot: ZfsSnapshot + :type raw: bool + """ + + ret = [] + + if self.snapshots: + if target_common_snapshot is None: + # No common snapshot found: every existing target snapshot is in + # the way and must be handled by handle_incompatible_snapshots. + return list(self.snapshots) + + followup = True + common_index = self.find_snapshot_index(target_common_snapshot) + assert common_index is not None + for snapshot in self.snapshots[common_index + 1:]: + if raw or not followup or int(snapshot.properties['written']) != 0: + followup = False + ret.append(snapshot) + + return ret + + @property + def written_since_ours(self): + """get number of bytes written since our last snapshot + :rtype: int + """ + + if self._written_since_ours is None: + latest_snapshot = self.our_snapshots[-1] + + self.debug("Getting bytes written since our last snapshot") + cmd = ["zfs", "get", "-H", "-ovalue", "-p", "written@" + str(latest_snapshot), self.name] + + output = cast("list[str]", self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0])) + + self._written_since_ours = int(output[0]) + + return self._written_since_ours + + def is_changed(self, min_changed_bytes=1): + """dataset is changed since ANY latest snapshot ? + + Args: + :type min_changed_bytes: int + """ + self.debug("Checking if dataset is changed {} or more bytes".format(min_changed_bytes)) + + if min_changed_bytes == 0: + return True + + if int(self.properties['written']) < min_changed_bytes: + return False + else: + return True + + def is_changed_ours(self, min_changed_bytes=1): + """dataset is changed since OUR latest snapshot? + + Args: + :type min_changed_bytes: int + """ + + if min_changed_bytes == 0: + return True + + if not self.our_snapshots: + return True + + # NOTE: filesystems can have a very small amount written without actual changes in some cases + if self.written_since_ours < min_changed_bytes: + return False + + return True + + def find_snapshot(self, snapshot_name): + """find snapshot by snapshot name (can be a suffix or a different + ZfsSnapshot) Returns None if it can't find it. + + Note that matches with our own snapshots will be done tagless. + + Args: + :rtype: ZfsSnapshot|None + :type snapshot_name: str|ZfsPointInTime|None + """ + + if snapshot_name is None: + return None + + if isinstance(snapshot_name, ZfsPointInTime): + tagless_suffix = snapshot_name.tagless_suffix + else: + tagless_suffix = snapshot_name + + for snapshot_name in self.snapshots: + if snapshot_name.tagless_suffix == tagless_suffix: + return snapshot_name + + return None + + def find_next_snapshot(self, snapshot_bookmark): + """find next snapshot in this dataset, according to snapshot or bookmark. None if it doesn't exist + Args: + :type snapshot_bookmark: ZfsSnapshot|ZfsBookmark + :rtype: ZfsSnapshot|None + + """ + + found = False + for snapshot in self.snapshots_bookmarks: + if snapshot == snapshot_bookmark: + found = True + else: + if found and type(snapshot) is ZfsSnapshot: + return snapshot + + return None + + def automount(self): + """Mount the dataset as if one did a zfs mount -a, but only for this dataset + Failure to mount doesn't result in an exception, but outputs errors to STDERR. + + """ + + self.debug("Auto mounting") + + if self.properties['type'] != "filesystem": + return + + if self.properties['canmount'] != 'on': + return + + if self.properties['mountpoint'] == 'legacy': + return + + if self.properties['mountpoint'] == 'none': + return + + if self.properties['encryption'] != 'off' and self.properties['keystatus'] == 'unavailable': + return + + self.zfs_node.run(["zfs", "mount", self.name], valid_exitcodes=[0, 1]) + + def abort_resume(self): + """abort current resume state""" + self.debug("Aborting resume") + self.zfs_node.run(["zfs", "recv", "-A", self.name]) + if not self.snapshots: + #abort of an initial resume + self.force_exists=False + + + def rollback(self): + """rollback to latest existing snapshot on this dataset""" + + + for snapshot in reversed(self.snapshots): + if snapshot.exists: + snapshot.rollback() + return + + def thin_list(self, keeps=None, ignores=None): + """determines list[snapshots] that should be kept or deleted based on + the thinning schedule. cull the herd! + + returns: ( keeps, obsoletes ) + + Args: + :param keeps: list[snapshots] to always keep (usually the last) + :param ignores: snapshots to completely ignore (usually incompatible target snapshots that are going to be destroyed anyway) + :type keeps: list[ZfsSnapshot] + :type ignores: list[ZfsSnapshot] + :rtype: tuple[list[ZfsSnapshot], list[ZfsSnapshot]] + """ + + if ignores is None: + ignores = [] + if keeps is None: + keeps = [] + + snapshots = [snapshot for snapshot in self.our_snapshots if snapshot not in ignores] + + return self.zfs_node.thin_list(snapshots, keep_snapshots=keeps) + + def thin(self, skip_holds=False): + """destroys snapshots according to thin_list, except last snapshot + + Args: + :type skip_holds: bool + """ + + (keeps, obsoletes) = self.thin_list(keeps=self.our_snapshots[-1:]) + for obsolete in obsoletes: + if skip_holds and obsolete.is_hold(): + obsolete.verbose("Keeping (common snapshot)") + else: + obsolete.destroy() + self.snapshots.remove(obsolete) + + @property + def recursive_datasets(self, types="filesystem,volume"): + """get all (non-snapshot) datasets recursively under us + + Args: + :type types: str + :rtype: list[ZfsContainer] + """ + + if self._recursive_datasets is None: + self.debug("Getting all recursive datasets under us") + + names = cast("list[str]", self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ + "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name + ])) + + self._recursive_datasets = cast("list[ZfsContainer]", + self.zfs_node.get_datasets(names[1:], force_exists=True)) + + return self._recursive_datasets + + @property + def datasets(self, types="filesystem,volume"): + """get all (non-snapshot) datasets directly under us + + Args: + :type types: str + :rtype: list[ZfsContainer] + + """ + + if self._datasets is None: + self.debug("Getting all datasets under us") + + names = cast("list[str]", self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ + "zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name + ])) + + self._datasets = cast("list[ZfsContainer]", + self.zfs_node.get_datasets(names[1:], force_exists=True)) + + return self._datasets + + def create_filesystem(self, parents=False, unmountable=True): + """create this container as a filesystem + + Args: + :type parents: bool + :type unmountable: bool + """ + + # recurse up + parent = self.parent + if parents and parent is not None and not parent.exists: + parent.create_filesystem(parents, unmountable) + + cmd = ["zfs", "create"] + + if unmountable: + cmd.extend(["-o", "canmount=off"]) + + cmd.append(self.name) + self.zfs_node.run(cmd) + + self.force_exists = True + + if self.zfs_node.readonly: + self._snapshots_bookmarks=[] + self.simulate_properties() + + def cache_snapshot_bookmark(self, snapshot, force=False): + """Update our snapshot and bookmark cache (if we have any). Use force if you want to force the caching, potentially triggering a zfs list + Args: + :type snapshot: ZfsSnapshot|ZfsBookmark + :type force: bool + """ + + if force: + self.snapshots_bookmarks.append(snapshot) + + elif self._snapshots_bookmarks is not None: + self._snapshots_bookmarks.append(snapshot) + + @property + def our_snapshots(self): + """get list[snapshots] creates by us of this dataset""" + ret = [] + + for snapshot in self.snapshots: + if snapshot.is_ours: + ret.append(snapshot) + + return ret + + def find_guid_bookmark_snapshot(self, guid): + """find the first bookmark or snapshot that matches, prefers bookmarks. + Args: + :type guid:str + :rtype: ZfsBookmark|ZfsSnapshot|None + """ + # Since this is slower, we only use it if the name matching with find_snapshot and find_bookmark doesn work. + + for bookmark in self.bookmarks: + if bookmark.properties['guid'] == guid: + return bookmark + + for snapshot in self.snapshots: + if snapshot.properties['guid'] == guid: + return snapshot + + return None + + def find_bookmark(self, snapshot_bookmark, preferred_tag): + """find bookmark by bookmark name (can be a suffix or a different + ZfsSnapshot or ZfsBookmark) Returns None if it can't find it. + + We try to find the bookmark with the preferred tag (which is usually a target path guid, to prevent conflicting bookmarks by multiple sends) + If that fails, we return any bookmark that matches (and ignore the tag) + + Args: + :rtype: ZfsBookmark|None + :type snapshot_bookmark: str|ZfsBookmark|ZfsSnapshot|None + :type preferred_tag: str + """ + + if snapshot_bookmark is None: + return None + + if not isinstance(snapshot_bookmark, ZfsDataset): + tagless_suffix = snapshot_bookmark + else: + tagless_suffix = snapshot_bookmark.tagless_suffix + + for snapshot_bookmark in self.bookmarks: + if snapshot_bookmark.tagless_suffix == tagless_suffix and snapshot_bookmark.tag == preferred_tag: + return snapshot_bookmark + + for snapshot_bookmark in self.bookmarks: + if snapshot_bookmark.tagless_suffix == tagless_suffix: + return snapshot_bookmark + + return None + + def find_exact_bookmark(self, bookmark_name): + """find exact bookmark name, or retruns none + + :rtype: ZfsBookmark|None + """ + + for snapshot_bookmark in self.bookmarks: + if snapshot_bookmark.name == bookmark_name: + return snapshot_bookmark + return None + + def find_snapshot_index(self, snapshot): + """find exact snapshot index by snapshot (can be a snapshot_name or + ZfsSnapshot) + + Args: + :type snapshot: str or ZfsSnapshot + :rtype: int|None + """ + + if not isinstance(snapshot, ZfsDataset): + snapshot_name = snapshot + else: + snapshot_name = snapshot.suffix + + index = 0 + for snapshot in self.snapshots: + if snapshot.suffix == snapshot_name: + return index + index = index + 1 + + return None + + def find_common_snapshot(self, target_dataset, guid_check, bookmark_tag): + """find latest common snapshot/bookmark between us and target returns None if its + an initial transfer. + + On the source it prefers the specified bookmark_name + + Args: + :rtype: tuple[ZfsSnapshot|ZfsBookmark, ZfsSnapshot] | tuple[None,None] + :returns: (source_common_snapshot, target_common_snapshot) + :type guid_check: bool + :type target_dataset: ZfsContainer + :type bookmark_tag: str + """ + + for target_snapshot in reversed(target_dataset.snapshots): + + # Prefer bookmarks to snapshots + source_bookmark = self.find_bookmark(target_snapshot, preferred_tag=bookmark_tag) + if source_bookmark: + if guid_check and source_bookmark.properties['guid'] != target_snapshot.properties['guid']: + source_bookmark.warning("Bookmark has mismatching GUID, ignoring.") + else: + source_bookmark.debug("Common bookmark") + return source_bookmark, target_snapshot + + # Source snapshot with same suffix? + source_snapshot = self.find_snapshot(target_snapshot) + if source_snapshot: + if guid_check and not source_snapshot.guid_matches(target_snapshot): + source_snapshot.warning("Snapshot has mismatching GUID, ignoring.") + else: + source_snapshot.debug("Common snapshot") + return source_snapshot, target_snapshot + + # Extensive GUID search (slower but works with all names) + source_bookmark_snapshot = self.find_guid_bookmark_snapshot(target_snapshot.properties['guid']) + if source_bookmark_snapshot is not None: + return source_bookmark_snapshot, target_snapshot + + return None, None + + def is_selected(self, value, source, inherited, exclude_paths, exclude_unchanged): + """determine if dataset should be selected for backup (called from + ZfsNode) + + Args: + :type exclude_paths: list[str] + :type value: str + :type source: str + :type inherited: bool + :type exclude_unchanged: int + + :param value: Value of the zfs property ("false"/"true"/"child"/parent/"-") + :param source: Source of the zfs property ("local"/"received", "-") + :param inherited: True of the value/source was inherited from a higher dataset. + + Returns: True : Selected + False: Excluded + None: No property found + """ + + # sanity checks + if source not in ["local", "received", "-"]: + # probably a program error in zfs-autobackup or new feature in zfs + raise (Exception( + "{} autobackup-property has illegal source: '{}' (possible BUG)".format(self.name, source))) + + if value not in ["false", "true", "child", "parent", "-"]: + # user error + raise (Exception( + "{} autobackup-property has illegal value: '{}'".format(self.name, value))) + + # non specified, ignore + if value == "-": + return None + + # only select childs of this dataset, ignore + if value == "child" and not inherited: + return False + + # only select parent, no childs, ignore + if value == "parent" and inherited: + return False + + # manually excluded by property + if value == "false": + self.verbose("Excluded") + return False + + # from here on the dataset is selected by property, now do additional exclusion checks + + # our path starts with one of the excluded paths? + for exclude_path in exclude_paths: + # if self.name.startswith(exclude_path): + if (self.name + "/").startswith(exclude_path + "/"): + # too noisy for verbose + self.debug("Excluded (path in exclude list)") + return False + + if not self.is_changed(exclude_unchanged): + self.verbose("Excluded (by --exclude-unchanged)") + return False + + self.verbose("Selected") + return True + + def destroy(self, fail_exception=False, **kwargs): + + self.verbose("Destroying") + return super().destroy(fail_exception=fail_exception) + + @property + def origin(self): + """The origin snapshot of this clone, or None if this dataset is not a clone or the origin cannot be parsed. + + :rtype: ZfsSnapshot|None + """ + + origin = self.properties.get('origin', '-') + if origin == '-': + return None + + if '@' not in origin: + self.warning("Cannot parse clone origin '{}'.".format(origin)) + return None + + return self.zfs_node.get_snapshot(origin) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index b930e7b..7ac21eb 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1,16 +1,41 @@ -import re -from datetime import datetime -import sys -import time +from typing import TYPE_CHECKING, cast -from .CachedProperty import CachedProperty from .ExecuteNode import ExecuteError +if TYPE_CHECKING: + from .ZfsNode import ZfsNode + +# +# Inheritance / overview graph (comment): +# +# ZfsDataset +# ├─ ZfsPointInTime (zfs_autobackup/ZfsPointInTime.py) +# │ ├─ ZfsSnapshot (zfs_autobackup/ZfsSnapshot.py) +# │ └─ ZfsBookmark (zfs_autobackup/ZfsBookmark.py) +# └─ ZfsContainer (zfs_autobackup/ZfsContainer.py) +# +# Summary (key responsibilities / notable members): +# - ZfsDataset (this file) +# - Base for all dataset types (filesystems, volumes, snapshots, bookmarks) +# - Key attributes: .zfs_node, .name, .force_exists +# - Important properties: .exists_check, .exists, .properties +# - Important methods: destroy(), invalidate_cache(), set(), inherit(), mount(), unmount() +# - ZfsPointInTime +# - Abstract intermediate for point-in-time objects (snapshots & bookmarks) +# - Key: .prefix, .suffix, .timestamp, .is_ours, .tagless_suffix, .tag +# - ZfsSnapshot +# - Snapshot-specific operations: send_pipe(), recv_pipe(), transfer_snapshot(), bookmark(), clone(), hold/release +# - ZfsBookmark +# - Bookmark-specific behavior (name split on "#", light destroy override) +# - ZfsContainer +# - Filesystem/volume container logic: snapshots/bookmarks management, automount, thin/thin_list, sync_snapshots, find_common_snapshot +## + class ZfsDataset: - """a zfs dataset (filesystem/volume/snapshot/clone) Note that a dataset - doesn't have to actually exist (yet/anymore) Also most properties are cached - for performance-reasons, but also to allow --test to function correctly. + """A generic ZFS dataset, this has all the common functions of zfs filesystems, volumes, snapshots and bookmarks. + + It is used as a base class for ZfsSnapshot, ZfsBookmark and ZfsContainer. This class should not be instantiated directly. """ # illegal properties per dataset type. these will be removed from --set-properties and --filter-properties @@ -22,61 +47,90 @@ class ZfsDataset: def __init__(self, zfs_node, name, force_exists=None): """ Args: - :type zfs_node: ZfsNode.ZfsNode + :type zfs_node: ZfsNode :type name: str :type force_exists: bool """ - self.zfs_node = zfs_node - self.name = name # full name - self._virtual_snapshots = [] - self.invalidate() + + # prevent direct instantiation + if type(self) is ZfsDataset: + raise TypeError( + "should not be instantiated directly.") + + self.zfs_node = zfs_node # type: ZfsNode + self.name = name # full actual name of dataset + self.force_exists = force_exists + # caching + self._exists_check = None # type: None|bool + self._properties = None # type: None|dict[str,str] + + def invalidate_cache(self): + """clear caches""" + self.force_exists = None + self._exists_check = None + self._properties = None + def __repr__(self): return "{}: {}".format(self.zfs_node, self.name) def __str__(self): + + # return f"{self.zfs_node}: {self.name} {'force_exists' if self.force_exists else ''}" return self.name - def __eq__(self, obj): - if not isinstance(obj, ZfsDataset): + def __eq__(self, dataset): + """compare the full name of the dataset""" + + if not isinstance(dataset, ZfsDataset): return False - return self.name == obj.name + return self.name == dataset.name + + def __hash__(self): + return hash(self.name) + + def guid_matches(self, other): + """Compare GUIDs between this dataset and other. Returns True if they match, + or if either side has an empty guid (test mode simulation — can't compare). + + :type other: ZfsDataset + :rtype: bool + """ + guid_self = self.properties.get('guid', '') + guid_other = other.properties.get('guid', '') + if guid_self == '' or guid_other == '': + return True + return guid_self == guid_other def verbose(self, txt): """ Args: :type txt: str """ - self.zfs_node.verbose("{}: {}".format(self.name, txt)) + self.zfs_node.verbose("{}: {}".format(self, txt)) def error(self, txt): """ Args: :type txt: str """ - self.zfs_node.error("{}: {}".format(self.name, txt)) + self.zfs_node.error("{}: {}".format(self, txt)) def warning(self, txt): """ Args: :type txt: str """ - self.zfs_node.warning("{}: {}".format(self.name, txt)) + self.zfs_node.warning("{}: {}".format(self, txt)) def debug(self, txt): """ Args: :type txt: str """ - self.zfs_node.debug("{}: {}".format(self.name, txt)) - - def invalidate(self): - """clear caches""" - CachedProperty.clear(self) - self.force_exists = None - self._virtual_snapshots = [] + self.zfs_node.debug("{}: {}".format(self, txt)) def split_path(self): """return the path elements as an array""" @@ -103,186 +157,17 @@ class ZfsDataset: return "/".join(self.split_path()[:-count]) @property - def filesystem_name(self): - """filesystem part of the name (before the @)""" - if self.is_snapshot: - (filesystem, snapshot) = self.name.split("@") - return filesystem - else: - return self.name - - @property - def snapshot_name(self): - """snapshot part of the name""" - if not self.is_snapshot: - raise (Exception("This is not a snapshot")) - - (filesystem, snapshot_name) = self.name.split("@") - return snapshot_name - - @property - def is_snapshot(self): - """true if this dataset is a snapshot""" - return self.name.find("@") != -1 - - @property - def is_excluded(self): - """true if this dataset is a snapshot and matches the exclude pattern""" - if not self.is_snapshot: - return False - - - for pattern in self.zfs_node.exclude_snapshot_patterns: - if pattern.search(self.name) is not None: - self.debug("Excluded (path matches snapshot exclude pattern)") - return True - - - - def is_selected(self, value, source, inherited, exclude_received, exclude_paths, exclude_unchanged): - """determine if dataset should be selected for backup (called from - ZfsNode) - - Args: - :type exclude_paths: list[str] - :type value: str - :type source: str - :type inherited: bool - :type exclude_received: bool - :type exclude_unchanged: int - - :param value: Value of the zfs property ("false"/"true"/"child"/parent/"-") - :param source: Source of the zfs property ("local"/"received", "-") - :param inherited: True of the value/source was inherited from a higher dataset. - - Returns: True : Selected - False: Excluded - None: No property found - """ - - # sanity checks - if source not in ["local", "received", "-"]: - # probably a program error in zfs-autobackup or new feature in zfs - raise (Exception( - "{} autobackup-property has illegal source: '{}' (possible BUG)".format(self.name, source))) - - if value not in ["false", "true", "child", "parent", "-"]: - # user error - raise (Exception( - "{} autobackup-property has illegal value: '{}'".format(self.name, value))) - - # non specified, ignore - if value == "-": - return None - - # only select childs of this dataset, ignore - if value == "child" and not inherited: - return False - - # only select parent, no childs, ignore - if value == "parent" and inherited: - return False - - # manually excluded by property - if value == "false": - self.verbose("Excluded") - return False - - # from here on the dataset is selected by property, now do additional exclusion checks - - # our path starts with one of the excluded paths? - for exclude_path in exclude_paths: - # if self.name.startswith(exclude_path): - if (self.name + "/").startswith(exclude_path + "/"): - # too noisy for verbose - self.debug("Excluded (path in exclude list)") - return False - - if source == "received": - if exclude_received: - self.verbose("Excluded (dataset already received)") - return False - - if not self.is_changed(exclude_unchanged): - self.verbose("Excluded (by --exclude-unchanged)") - return False - - self.verbose("Selected") - return True - - @property - def parent(self): - """get zfs-parent of this dataset. for snapshots this means it will get - the filesystem/volume that it belongs to. otherwise it will return the - parent according to path - - we cache this so everything in the parent that is cached also stays. - - returns None if there is no parent. - :rtype: ZfsDataset | None - """ - if self.is_snapshot: - return self.zfs_node.get_dataset(self.filesystem_name) - else: - stripped = self.rstrip_path(1) - if stripped: - return self.zfs_node.get_dataset(stripped) - else: - return None - - # NOTE: unused for now - # def find_prev_snapshot(self, snapshot, also_other_snapshots=False): - # """find previous snapshot in this dataset. None if it doesn't exist. - # - # also_other_snapshots: set to true to also return snapshots that where - # not created by us. (is_ours) - # - # Args: - # :type snapshot: str or ZfsDataset.ZfsDataset - # :type also_other_snapshots: bool - # """ - # - # if self.is_snapshot: - # raise (Exception("Please call this on a dataset.")) - # - # index = self.find_snapshot_index(snapshot) - # while index: - # index = index - 1 - # if also_other_snapshots or self.snapshots[index].is_ours(): - # return self.snapshots[index] - # return None - - def find_next_snapshot(self, snapshot, also_other_snapshots=False, clones='never'): - """find next snapshot in this dataset. None if it doesn't exist - - Args: - :type snapshot: ZfsDataset - :type also_other_snapshots: bool - :type clones: str - """ - - if self.is_snapshot: - raise (Exception("Please call this on a dataset.")) - - if clones != 'never' and snapshot.name == self.properties.get("origin"): - # Special case when start snapshot filesystem is other - index = -1 - else: - index = self.find_snapshot_index(snapshot) - - while index is not None and index < len(self.snapshots) - 1: - index = index + 1 - if also_other_snapshots or self.snapshots[index].is_ours(): - return self.snapshots[index] - return None - - @CachedProperty def exists_check(self): """check on disk if it exists""" - self.debug("Checking if dataset exists") - return (len(self.zfs_node.run(tab_split=True, cmd=["zfs", "list", self.name], readonly=True, - valid_exitcodes=[0, 1], - hide_errors=True)) > 0) + + if self._exists_check is None: + self.debug("Checking if dataset exists") + output = cast("list[str]", self.zfs_node.run( + tab_split=True, cmd=["zfs", "list", self.name], readonly=True, + valid_exitcodes=[0, 1], hide_errors=True)) + self._exists_check = (len(output) > 0) + + return self._exists_check @property def exists(self): @@ -299,50 +184,22 @@ class ZfsDataset: else: return self.exists_check - def create_filesystem(self, parents=False, unmountable=True): - """create a filesystem - - Args: - :type parents: bool - """ - - # recurse up - if parents and self.parent and not self.parent.exists: - self.parent.create_filesystem(parents, unmountable) - - cmd = ["zfs", "create"] - - if unmountable: - cmd.extend(["-o", "canmount=off"]) - - cmd.append(self.name) - self.zfs_node.run(cmd) - - self.force_exists = True - - def destroy(self, fail_exception=False, deferred=False, verbose=True): + def destroy(self, fail_exception=False, deferred=False): """destroy the dataset. by default failures are not an exception, so we can continue making backups Args: + :type deferred: bool :type fail_exception: bool """ - if verbose: - self.verbose("Destroying") - else: - self.debug("Destroying") - - if self.is_snapshot: - self.release() - try: - if deferred and self.is_snapshot: + if deferred: self.zfs_node.run(["zfs", "destroy", "-d", self.name]) else: self.zfs_node.run(["zfs", "destroy", self.name]) - self.invalidate() + self.invalidate_cache() self.force_exists = False return True except ExecuteError: @@ -351,626 +208,30 @@ class ZfsDataset: else: raise - @CachedProperty + @property def properties(self): - """all zfs properties""" - - cmd = [ - "zfs", "get", "-H", "-o", "property,value", "-p", "all", self.name - ] - - self.debug("Getting zfs properties") - - ret = {} - for pair in self.zfs_node.run(tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]): - if len(pair) == 2: - ret[pair[0]] = pair[1] - - return ret - - def is_changed(self, min_changed_bytes=1): - """dataset is changed since ANY latest snapshot ? - - Args: - :type min_changed_bytes: int - """ - self.debug("Checking if dataset is changed") - - if min_changed_bytes == 0: - return True - - if int(self.properties['written']) < min_changed_bytes: - return False - else: - return True - - def is_ours(self): - """return true if this snapshot name has format""" - try: - test = self.timestamp - except ValueError as e: - return False - - return True - - @property - def holds(self): - """get list[holds] for dataset""" - - output = self.zfs_node.run(["zfs", "holds", "-H", self.name], valid_exitcodes=[0], tab_split=True, - readonly=True) - return map(lambda fields: fields[1], output) - - def is_hold(self): - """did we hold this snapshot?""" - return self.zfs_node.hold_name in self.holds - - def hold(self): - """hold dataset""" - self.debug("holding") - self.zfs_node.run(["zfs", "hold", self.zfs_node.hold_name, self.name], valid_exitcodes=[0, 1]) - - def release(self): - """release dataset""" - if self.zfs_node.readonly or self.is_hold(): - self.debug("releasing") - self.zfs_node.run(["zfs", "release", self.zfs_node.hold_name, self.name], valid_exitcodes=[0, 1]) - - @property - def timestamp(self): - """get timestamp from snapshot name. Only works for our own snapshots - with the correct format. - """ - dt = datetime.strptime(self.snapshot_name, self.zfs_node.snapshot_time_format) - if sys.version_info[0] >= 3: - from datetime import timezone - if self.zfs_node.utc: - dt = dt.replace(tzinfo=timezone.utc) - seconds = dt.timestamp() - else: - # python2 has no good functions to deal with UTC. Yet the unix timestamp - # must be in UTC to allow comparison against `time.time()` in on other parts - # of this project (e.g. Thinner.py). If we are handling UTC timestamps, - # we must adjust for that here. - if self.zfs_node.utc: - seconds = (dt - datetime(1970, 1, 1)).total_seconds() - else: - seconds = time.mktime(dt.timetuple()) - return seconds - - def from_names(self, names, force_exists=None): - """convert a list[names] to a list ZfsDatasets for this zfs_node - - Args: - :type names: list[str] - """ - ret = [] - for name in names: - ret.append(self.zfs_node.get_dataset(name, force_exists)) - - return ret - - # def add_virtual_snapshot(self, snapshot): - # """pretend a snapshot exists (usefull in test mode)""" - # - # # NOTE: we could just call self.snapshots.append() but this would trigger a zfs list which is not always needed. - # if CachedProperty.is_cached(self, 'snapshots'): - # # already cached so add it - # print ("ADDED") - # self.snapshots.append(snapshot) - # else: - # # self.snapshots will add it when requested - # print ("ADDED VIRT") - # self._virtual_snapshots.append(snapshot) - - @CachedProperty - def snapshots(self): - """get all snapshots of this dataset - :rtype: ZfsDataset + """all zfs properties + :rtype: dict[str, str] """ - #FIXME: dont check for existance. (currenlty needed for _add_virtual_snapshots) - if not self.exists: - return [] + if self._properties is None: - self.debug("Getting snapshots") + cmd = [ + "zfs", "get", "-H", "-o", "property,value", "-p", "all", self.name + ] - cmd = [ - "zfs", "list", "-d", "1", "-r", "-t", "snapshot", "-H", "-o", "name", self.name - ] + self.debug("Getting zfs properties") + properties = {} # type: dict[str, str] - return self.from_names(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) + output = cast("list[list[str]]", cast(object, self.zfs_node.run( + tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]))) + for pair in output: + if len(pair) == 2: + properties[pair[0]] = pair[1] + self._properties = properties - @property - def our_snapshots(self): - """get list[snapshots] creates by us of this dataset""" - ret = [] - for snapshot in self.snapshots: - if snapshot.is_ours(): - ret.append(snapshot) + return self._properties - return ret - - def find_snapshot(self, snapshot): - """find snapshot by snapshot (can be a snapshot_name or a different - ZfsDataset ) - - Args: - :rtype: ZfsDataset - :type snapshot: str or ZfsDataset - """ - - if not isinstance(snapshot, ZfsDataset): - snapshot_name = snapshot - else: - snapshot_name = snapshot.snapshot_name - - for snapshot in self.snapshots: - if snapshot.snapshot_name == snapshot_name: - return snapshot - - return None - - def find_snapshot_index(self, snapshot): - """find snapshot index by snapshot (can be a snapshot_name or - ZfsDataset) - - Args: - :type snapshot: str or ZfsDataset - """ - - if not isinstance(snapshot, ZfsDataset): - snapshot_name = snapshot - else: - snapshot_name = snapshot.snapshot_name - - index = 0 - for snapshot in self.snapshots: - if snapshot.snapshot_name == snapshot_name: - return index - index = index + 1 - - return None - - @CachedProperty - def written_since_ours(self): - """get number of bytes written since our last snapshot""" - - latest_snapshot = self.our_snapshots[-1] - - self.debug("Getting bytes written since our last snapshot") - cmd = ["zfs", "get", "-H", "-ovalue", "-p", "written@" + str(latest_snapshot), self.name] - - output = self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0]) - - return int(output[0]) - - def is_changed_ours(self, min_changed_bytes=1): - """dataset is changed since OUR latest snapshot? - - Args: - :type min_changed_bytes: int - """ - - if min_changed_bytes == 0: - return True - - if not self.our_snapshots: - return True - - # NOTE: filesystems can have a very small amount written without actual changes in some cases - if self.written_since_ours < min_changed_bytes: - return False - - return True - - @CachedProperty - def recursive_datasets(self, types="filesystem,volume"): - """get all (non-snapshot) datasets recursively under us - - Args: - :type types: str - """ - - self.debug("Getting all recursive datasets under us") - - names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ - "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name - ]) - - return self.from_names(names[1:], force_exists=True) - - @CachedProperty - def datasets(self, types="filesystem,volume"): - """get all (non-snapshot) datasets directly under us - - Args: - :type types: str - """ - - self.debug("Getting all datasets under us") - - names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ - "zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name - ]) - - return self.from_names(names[1:], force_exists=True) - - def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, write_embedded, - send_pipes, zfs_compressed): - """returns a pipe with zfs send output for this snapshot - - resume_token: resume sending from this token. (in that case we don't - need to know snapshot names) - - Args: - :param send_pipes: output cmd array that will be added to actual zfs send command. (e.g. mbuffer or compression program) - :type send_pipes: list[str] - :type features: list[str] - :type prev_snapshot: ZfsDataset - :type resume_token: str - :type show_progress: bool - :type raw: bool - """ - # build source command - cmd = [] - - cmd.extend(["zfs", "send", ]) - - # all kind of performance options: - if 'large_blocks' in features and "-L" in self.zfs_node.supported_send_options: - # large block support (only if recordsize>128k which is seldomly used) - cmd.append("-L") # --large-block - - if write_embedded and 'embedded_data' in features and "-e" in self.zfs_node.supported_send_options: - cmd.append("-e") # --embed; WRITE_EMBEDDED, more compact stream - - if zfs_compressed and "-c" in self.zfs_node.supported_send_options: - cmd.append("-c") # --compressed; use compressed WRITE records - - # raw? (send over encrypted data in its original encrypted form without decrypting) - if raw: - cmd.append("--raw") - - # progress output - if show_progress: - cmd.append("-v") # --verbose - cmd.append("-P") # --parsable - - # resume a previous send? (don't need more parameters in that case) - if resume_token: - cmd.extend(["-t", resume_token]) - - else: - # send properties - if send_properties: - cmd.append("-p") # --props - - # incremental? - if prev_snapshot: - if self.filesystem_name == prev_snapshot.filesystem_name: - cmd.extend(["-i", "@" + prev_snapshot.snapshot_name]) - else: - cmd.extend(["-i", prev_snapshot.name]) - - cmd.append(self.name) - - cmd.extend(send_pipes) - - output_pipe = self.zfs_node.run(cmd, pipe=True, readonly=True) - - return output_pipe - - def recv_pipe(self, pipe, features, recv_pipes, filter_properties=None, set_properties=None, ignore_exit_code=False, - force=False): - """starts a zfs recv for this snapshot and uses pipe as input - - note: you can it both on a snapshot or filesystem object. The - resulting zfs command is the same, only our object cache is invalidated - differently. - - Args: - :param recv_pipes: input cmd array that will be prepended to actual zfs recv command. (e.g. mbuffer or decompression program) - :type pipe: subprocess.pOpen - :type features: list[str] - :type filter_properties: list[str] - :type set_properties: list[str] - :type ignore_exit_code: bool - """ - - if set_properties is None: - set_properties = [] - - if filter_properties is None: - filter_properties = [] - - # build target command - cmd = [] - - cmd.extend(recv_pipes) - - cmd.extend(["zfs", "recv"]) - - # don't let zfs recv mount everything thats received (even with canmount=noauto!) - cmd.append("-u") - - for property_ in filter_properties: - cmd.extend(["-x", property_]) - - for property_ in set_properties: - cmd.extend(["-o", property_]) - - # verbose output - cmd.append("-v") - - if force: - cmd.append("-F") - - if 'extensible_dataset' in features and "-s" in self.zfs_node.supported_recv_options: - # support resuming - self.debug("Enabled resume support") - cmd.append("-s") - - cmd.append(self.filesystem_name) - - if ignore_exit_code: - valid_exitcodes = [] - else: - valid_exitcodes = [0] - - # self.zfs_node.reset_progress() - self.zfs_node.run(cmd, inp=pipe, valid_exitcodes=valid_exitcodes) - - # invalidate cache - self.invalidate() - - # in test mode we assume everything was ok and it exists - if self.zfs_node.readonly: - self.force_exists = True - - # check if transfer was really ok (exit codes have been wrong before due to bugs in zfs-utils and some - # errors should be ignored, thats where the ignore_exitcodes is for.) - if not self.exists: - self.error("error during transfer") - raise (Exception("Target doesn't exist after transfer, something went wrong.")) - - # at this point we're sure the actual dataset exists - self.parent.force_exists = True - - def automount(self): - """Mount the dataset as if one did a zfs mount -a, but only for this dataset - Failure to mount doesnt result in an exception, but outputs errors to STDERR. - - """ - - self.debug("Auto mounting") - - if self.properties['type'] != "filesystem": - return - - if self.properties['canmount'] != 'on': - return - - if self.properties['mountpoint'] == 'legacy': - return - - if self.properties['mountpoint'] == 'none': - return - - if self.properties['encryption'] != 'off' and self.properties['keystatus'] == 'unavailable': - return - - self.zfs_node.run(["zfs", "mount", self.name], valid_exitcodes=[0,1]) - - def transfer_snapshot(self, target_snapshot, features, prev_snapshot, show_progress, - filter_properties, set_properties, ignore_recv_exit_code, resume_token, - raw, send_properties, write_embedded, send_pipes, recv_pipes, zfs_compressed, force): - """transfer this snapshot to target_snapshot. specify prev_snapshot for - incremental transfer - - connects a send_pipe() to recv_pipe() - - Args: - :type send_pipes: list[str] - :type recv_pipes: list[str] - :type target_snapshot: ZfsDataset - :type features: list[str] - :type prev_snapshot: ZfsDataset - :type show_progress: bool - :type filter_properties: list[str] - :type set_properties: list[str] - :type ignore_recv_exit_code: bool - :type resume_token: str - :type raw: bool - """ - - if set_properties is None: - set_properties = [] - if filter_properties is None: - filter_properties = [] - - self.debug("Transfer snapshot to {}".format(target_snapshot.filesystem_name)) - - if resume_token: - self.verbose("resuming") - - # initial or increment - if not prev_snapshot: - self.verbose("-> {} (new)".format(target_snapshot.filesystem_name)) - else: - # incremental - self.verbose("-> {}".format(target_snapshot.filesystem_name)) - - # do it - pipe = self.send_pipe(features=features, show_progress=show_progress, prev_snapshot=prev_snapshot, - resume_token=resume_token, raw=raw, send_properties=send_properties, - write_embedded=write_embedded, send_pipes=send_pipes, zfs_compressed=zfs_compressed) - target_snapshot.recv_pipe(pipe, features=features, filter_properties=filter_properties, - set_properties=set_properties, ignore_exit_code=ignore_recv_exit_code, - recv_pipes=recv_pipes, force=force) - - # try to automount it, if its the initial transfer - if not prev_snapshot: - # in test mode it doesnt actually exist, so dont try to mount it/read properties - if not target_snapshot.zfs_node.readonly: - target_snapshot.parent.automount() - - def abort_resume(self): - """abort current resume state""" - self.debug("Aborting resume") - self.zfs_node.run(["zfs", "recv", "-A", self.name]) - - def rollback(self): - """rollback to latest existing snapshot on this dataset""" - - for snapshot in reversed(self.snapshots): - if snapshot.exists: - self.debug("Rolling back") - self.zfs_node.run(["zfs", "rollback", snapshot.name]) - return - - def get_resume_snapshot(self, resume_token): - """returns snapshot that will be resumed by this resume token (run this - on source with target-token) - - Args: - :type resume_token: str - """ - # use zfs send -n option to determine this - # NOTE: on smartos stderr, on linux stdout - (stdout, stderr) = self.zfs_node.run(["zfs", "send", "-t", resume_token, "-n", "-v"], valid_exitcodes=[0, 255], - readonly=True, return_stderr=True) - if stdout: - lines = stdout - else: - lines = stderr - for line in lines: - matches = re.findall("toname = .*@(.*)", line) - if matches: - snapshot_name = matches[0] - snapshot = self.zfs_node.get_dataset(self.filesystem_name + "@" + snapshot_name) - snapshot.debug("resume token belongs to this snapshot") - return snapshot - - return None - - def thin_list(self, keeps=None, ignores=None): - """determines list[snapshots] that should be kept or deleted based on - the thinning schedule. cull the herd! - - returns: ( keeps, obsoletes ) - - Args: - :param keeps: list[snapshots] to always keep (usually the last) - :param ignores: snapshots to completely ignore (usually incompatible target snapshots that are going to be destroyed anyway) - :type keeps: list[ZfsDataset] - :type ignores: list[ZfsDataset] - """ - - if ignores is None: - ignores = [] - if keeps is None: - keeps = [] - - snapshots = [snapshot for snapshot in self.our_snapshots if snapshot not in ignores] - - return self.zfs_node.thin(snapshots, keep_objects=keeps) - - def thin(self, skip_holds=False): - """destroys snapshots according to thin_list, except last snapshot - - Args: - :type skip_holds: bool - """ - - (keeps, obsoletes) = self.thin_list(keeps=self.our_snapshots[-1:]) - for obsolete in obsoletes: - if skip_holds and obsolete.is_hold(): - obsolete.verbose("Keeping (common snapshot)") - else: - obsolete.destroy() - self.snapshots.remove(obsolete) - - def find_common_snapshot(self, target_dataset, guid_check, clones='never'): - """find latest common snapshot between us and target returns None if its - an initial transfer - - Args: - :type guid_check: bool - :type target_dataset: ZfsDataset - """ - - if not target_dataset.snapshots: - # target has nothing yet - origin = self.properties.get("origin") - if origin: - if clones == 'never': - self.warning("Clones support is disabled, expect dataset expansion") - return None - # We are a clone. The origin has earlier createtxg, and thus - # must have been already synced. - return ZfsDataset(self.zfs_node, origin) - return None - else: - for source_snapshot in reversed(self.snapshots): - target_snapshot = target_dataset.find_snapshot(source_snapshot) - if target_snapshot: - if guid_check and source_snapshot.properties['guid'] != target_snapshot.properties['guid']: - target_snapshot.warning("Common snapshot has invalid guid, ignoring.") - else: - target_snapshot.debug("common snapshot") - return source_snapshot - # target_dataset.error("Cant find common snapshot with source.") - raise (Exception("Cant find common snapshot with target.")) - - def find_start_snapshot(self, common_snapshot, also_other_snapshots, clones='never'): - """finds first snapshot to send :rtype: ZfsDataset or None if we cant - find it. - - Args: - :type common_snapshot: ZfsDataset - :type also_other_snapshots: bool - :type clones: str - """ - - if not common_snapshot: - if not self.snapshots: - start_snapshot = None - else: - # no common snapshot, start from beginning - start_snapshot = self.snapshots[0] - - if not start_snapshot.is_ours() and not also_other_snapshots: - # try to start at a snapshot thats ours - start_snapshot = self.find_next_snapshot(start_snapshot, also_other_snapshots, clones=clones) - else: - # normal situation: start_snapshot is the one after the common snapshot - start_snapshot = self.find_next_snapshot(common_snapshot, also_other_snapshots, clones=clones) - - return start_snapshot - - def find_incompatible_snapshots(self, common_snapshot, raw): - """returns a list[snapshots] that is incompatible for a zfs recv onto - the common_snapshot. all direct followup snapshots with written=0 are - compatible. - - in raw-mode nothing is compatible. issue #219 - - Args: - :type common_snapshot: ZfsDataset - :type raw: bool - """ - - ret = [] - - if common_snapshot and self.snapshots: - followup = True - for snapshot in self.snapshots[self.find_snapshot_index(common_snapshot) + 1:]: - if raw or not followup or int(snapshot.properties['written']) != 0: - followup = False - ret.append(snapshot) - - return ret def get_allowed_properties(self, filter_properties, set_properties): """only returns lists of allowed properties for this dataset type @@ -994,280 +255,36 @@ class ZfsDataset: return allowed_filter_properties, allowed_set_properties - def _add_virtual_snapshots(self, source_dataset, source_start_snapshot, also_other_snapshots): - """add snapshots from source to our snapshot list. (just the in memory - list, no disk operations) + def set(self, prop, value): + """set a zfs property""" - Args: - :type source_dataset: ZfsDataset - :type source_start_snapshot: ZfsDataset - :type also_other_snapshots: bool - """ + self.debug("Setting {}={}".format(prop, value)) - self.debug("Creating virtual target snapshots") - snapshot = source_start_snapshot - while snapshot: - # create virtual target snapsho - # NOTE: with force_exist we're telling the dataset it doesnt exist yet. (e.g. its virtual) - virtual_snapshot = self.zfs_node.get_dataset(self.filesystem_name + "@" + snapshot.snapshot_name, - force_exists=False) - self.snapshots.append(virtual_snapshot) - snapshot = source_dataset.find_next_snapshot(snapshot, also_other_snapshots) + cmd = [ + "zfs", "set", "{}={}".format(prop, value), self.name + ] - def _pre_clean(self, common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_keeps): - """cleanup old stuff before starting snapshot syncing + self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - Args: - :type common_snapshot: ZfsDataset - :type target_dataset: ZfsDataset - :type source_obsoletes: list[ZfsDataset] - :type target_obsoletes: list[ZfsDataset] - :type target_keeps: list[ZfsDataset] - """ + # invalidate cache + self._properties = None - # on source: destroy all obsoletes before common. (since we cant send them anyways) - # But after common, only delete snapshots that target also doesn't want - if common_snapshot: - before_common = True - else: - before_common = False + def inherit(self, prop): + """inherit zfs property""" - for source_snapshot in self.snapshots: - if common_snapshot and source_snapshot.snapshot_name == common_snapshot.snapshot_name: - before_common = False - # never destroy common snapshot - else: - target_snapshot = target_dataset.find_snapshot(source_snapshot) - if (source_snapshot in source_obsoletes) and (before_common or (target_snapshot not in target_keeps)): - source_snapshot.destroy() + self.debug("Inheriting property {}".format(prop)) - # on target: destroy everything thats obsolete, except common_snapshot - for target_snapshot in target_dataset.snapshots: - if (target_snapshot in target_obsoletes) \ - and (not common_snapshot or (target_snapshot.snapshot_name != common_snapshot.snapshot_name)): - if target_snapshot.exists: - target_snapshot.destroy() + cmd = [ + "zfs", "inherit", prop, self.name + ] - def _validate_resume_token(self, target_dataset, start_snapshot): - """validate and get (or destory) resume token + self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - Args: - :type target_dataset: ZfsDataset - :type start_snapshot: ZfsDataset - """ - - if target_dataset.exists and 'receive_resume_token' in target_dataset.properties: - if start_snapshot == None: - target_dataset.verbose("Aborting resume, its obsolete.") - target_dataset.abort_resume() - else: - resume_token = target_dataset.properties['receive_resume_token'] - # not valid anymore - resume_snapshot = self.get_resume_snapshot(resume_token) - if not resume_snapshot or start_snapshot.snapshot_name != resume_snapshot.snapshot_name: - target_dataset.verbose("Aborting resume, its no longer valid.") - target_dataset.abort_resume() - else: - return resume_token - - def _plan_sync(self, target_dataset, also_other_snapshots, guid_check, raw, clones='never'): - """plan where to start syncing and what to sync and what to keep - - Args: - :rtype: ( ZfsDataset, ZfsDataset, list[ZfsDataset], list[ZfsDataset], list[ZfsDataset], list[ZfsDataset] ) - :type target_dataset: ZfsDataset - :type also_other_snapshots: bool - :type guid_check: bool - :type raw: bool - :type clones: str - """ - - # determine common and start snapshot - target_dataset.debug("Determining start snapshot") - common_snapshot = self.find_common_snapshot(target_dataset, guid_check=guid_check, clones=clones) - start_snapshot = self.find_start_snapshot(common_snapshot, also_other_snapshots, clones=clones) - incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(common_snapshot, raw) - - # let thinner decide whats obsolete on source - source_obsoletes = [] - if self.our_snapshots: - source_obsoletes = self.thin_list(keeps=[self.our_snapshots[-1]])[1] - - # let thinner decide keeps/obsoletes on target, AFTER the transfer would be done (by using virtual snapshots) - target_dataset._add_virtual_snapshots(self, start_snapshot, also_other_snapshots) - target_keeps = [] - target_obsoletes = [] - if target_dataset.our_snapshots: - (target_keeps, target_obsoletes) = target_dataset.thin_list(keeps=[target_dataset.our_snapshots[-1]], - ignores=incompatible_target_snapshots) - - return common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_keeps, incompatible_target_snapshots - - def handle_incompatible_snapshots(self, incompatible_target_snapshots, destroy_incompatible): - """destroy incompatbile snapshots on target before sync, or inform user - what to do - - Args: - :type incompatible_target_snapshots: list[ZfsDataset] - :type destroy_incompatible: bool - """ - - if incompatible_target_snapshots: - if not destroy_incompatible: - for snapshot in incompatible_target_snapshots: - snapshot.error("Incompatible snapshot") - raise (Exception("Please destroy incompatible snapshots on target, or use --destroy-incompatible.")) - else: - for snapshot in incompatible_target_snapshots: - snapshot.verbose("Incompatible snapshot") - snapshot.destroy(fail_exception=True) - self.snapshots.remove(snapshot) - - if len(incompatible_target_snapshots) > 0: - self.rollback() - - def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, - ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, - no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, - clones, make_target_name): - """sync this dataset's snapshots to target_dataset, while also thinning - out old snapshots along the way. - - Args: - :type send_pipes: list[str] - :type recv_pipes: list[str] - :type target_dataset: ZfsDataset - :type features: list[str] - :type show_progress: bool - :type filter_properties: list[str] - :type set_properties: list[str] - :type ignore_recv_exit_code: bool - :type holds: bool - :type rollback: bool - :type decrypt: bool - :type also_other_snapshots: bool - :type no_send: bool - :type guid_check: bool - :type clones: str - :type make_target_name: Callable[[ZfsDataset], str] - """ - - # self.verbose("-> {}".format(target_dataset)) - - # defaults for these settings if there is no encryption stuff going on: - send_properties = True - raw = False - write_embedded = True - - # source dataset encrypted? - if self.properties.get('encryption', 'off') != 'off': - # user wants to send it over decrypted? - if decrypt: - # when decrypting, zfs cant send properties - send_properties = False - else: - # keep data encrypted by sending it raw (including properties) - raw = True - - (common_snapshot, start_snapshot, source_obsoletes, target_obsoletes, target_keeps, - incompatible_target_snapshots) = \ - self._plan_sync(target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, - guid_check=guid_check, raw=raw, clones=clones) - - target_origin = None - if (clones != 'never' - and not target_dataset.exists - and common_snapshot - and common_snapshot.filesystem_name != target_dataset.filesystem_name - ): - target_origin = ZfsDataset(target_dataset.zfs_node, make_target_name(common_snapshot)) - if not target_origin.exists: - raise Exception("Origin {} for clone {} does not exist on target. You may want to retransfer {} by other means." - .format(target_origin.name, target_dataset.name, common_snapshot.name)) - - # NOTE: we do this because we dont want filesystems to fillup when backups keep failing. - # Also usefull with no_send to still cleanup stuff. - self._pre_clean( - common_snapshot=common_snapshot, target_dataset=target_dataset, - target_keeps=target_keeps, target_obsoletes=target_obsoletes, source_obsoletes=source_obsoletes) - - # handle incompatible stuff on target - target_dataset.handle_incompatible_snapshots(incompatible_target_snapshots, destroy_incompatible) - - # now actually transfer the snapshots, if we want - if no_send: - return - - # check if we can resume - resume_token = self._validate_resume_token(target_dataset, start_snapshot) - - (active_filter_properties, active_set_properties) = self.get_allowed_properties(filter_properties, - set_properties) - - # encrypt at target? - if encrypt and not raw: - # filter out encryption properties to let encryption on the target take place - active_filter_properties.extend(["keylocation", "pbkdf2iters", "keyformat", "encryption"]) - write_embedded = False - - # now actually transfer the snapshots - prev_source_snapshot = common_snapshot - source_snapshot = start_snapshot - do_rollback = rollback - while source_snapshot: - target_snapshot = target_dataset.find_snapshot(source_snapshot) # still virtual - - # does target actually want it? - if target_snapshot not in target_obsoletes and not source_snapshot.is_excluded: - - # do the rollback, one time at first transfer - if do_rollback: - target_dataset.rollback() - do_rollback = False - - source_snapshot.transfer_snapshot(target_snapshot, features=features, - prev_snapshot=prev_source_snapshot, show_progress=show_progress, - filter_properties=active_filter_properties, - set_properties=active_set_properties, - ignore_recv_exit_code=ignore_recv_exit_code, - resume_token=resume_token, write_embedded=write_embedded, raw=raw, - send_properties=send_properties, send_pipes=send_pipes, - recv_pipes=recv_pipes, zfs_compressed=zfs_compressed, force=force) - - resume_token = None - - # hold the new common snapshots and release the previous ones - if holds: - target_snapshot.hold() - source_snapshot.hold() - - if prev_source_snapshot: - if holds: - prev_source_snapshot.release() - (target_origin or target_dataset.find_snapshot(prev_source_snapshot)).release() - - # we may now destroy the previous source snapshot if its obsolete - if prev_source_snapshot in source_obsoletes: - prev_source_snapshot.destroy() - - # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, - # the rest was already destroyed or will not be send) - prev_target_snapshot = target_dataset.find_snapshot(prev_source_snapshot) - if prev_target_snapshot in target_obsoletes: - prev_target_snapshot.destroy() - - prev_source_snapshot = source_snapshot - else: - source_snapshot.debug("skipped (target doesn't need it)") - # was it actually a resume? - if resume_token: - target_dataset.verbose("Aborting resume, we dont want that snapshot anymore.") - target_dataset.abort_resume() - resume_token = None - - source_snapshot = self.find_next_snapshot(source_snapshot, also_other_snapshots, clones=clones) + # invalidate cache + self._properties = None def mount(self, mount_point): + """Mount the container or snapshot at mount_point, if it is a filesystem.""" self.debug("Mounting") @@ -1287,41 +304,9 @@ class ZfsDataset: self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - def clone(self, name): - """clones this snapshot and returns ZfsDataset of the clone""" - - self.debug("Cloning to {}".format(name)) - - cmd = [ - "zfs", "clone", self.name, name - ] - - self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - - return self.zfs_node.get_dataset(name, force_exists=True) - - def set(self, prop, value): - """set a zfs property""" - - self.debug("Setting {}={}".format(prop, value)) - - cmd = [ - "zfs", "set", "{}={}".format(prop, value), self.name - ] - - self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - - self.invalidate() - - def inherit(self, prop): - """inherit zfs property""" - - self.debug("Inheriting property {}".format(prop)) - - cmd = [ - "zfs", "inherit", prop, self.name - ] - - self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - - self.invalidate() + def simulate_properties(self): + """in test mode when we create fake snapshots or bookmarks, we also need to simluate properties""" + if self._properties is None: + self._properties={ + 'guid': '' + } diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index b1b2f43..8fcdc83 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -1,36 +1,45 @@ # python 2 compatibility from __future__ import print_function -from operator import attrgetter import re import shlex -import subprocess -import sys + import time from .ExecuteNode import ExecuteNode -from .Thinner import Thinner -from .CachedProperty import CachedProperty +from .ZfsContainer import ZfsContainer from .ZfsPool import ZfsPool from .ZfsDataset import ZfsDataset +from .ZfsSnapshot import ZfsSnapshot +from .ZfsBookmark import ZfsBookmark from .ExecuteNode import ExecuteError from .util import datetime_now + class ZfsNode(ExecuteNode): """a node that contains zfs datasets. implements global (systemwide/pool wide) zfs commands""" - def __init__(self, logger, utc=False, snapshot_time_format="", hold_name="", ssh_config=None, ssh_to=None, readonly=False, - description="", - debug_output=False, thinner=None, exclude_snapshot_patterns=[]): + # def __init__(self, logger, utc=False, snapshot_time_format="", hold_name="", ssh_config=None, ssh_to=None, + # readonly=False, + # description="", + # debug_output=False, thinner=None, exclude_snapshot_patterns=None, tag_seperator='~'): + def __init__(self, logger, utc, snapshot_time_format, hold_name, ssh_config, ssh_to, + readonly, + description, + debug_output, thinner, exclude_snapshot_patterns, tag_seperator): self.utc = utc self.snapshot_time_format = snapshot_time_format + self.tag_seperator = tag_seperator self.hold_name = hold_name self.description = description self.logger = logger + self._supported_send_options = None + self._supported_recv_options = None + self.exclude_snapshot_patterns = exclude_snapshot_patterns if ssh_config: @@ -49,46 +58,56 @@ class ZfsNode(ExecuteNode): else: self.verbose("Keep no old snaphots") - self.__thinner = thinner + self._thinner = thinner # list of ZfsPools - self.__pools = {} - self.__datasets = {} + self._pools = {} # type: dict[str, ZfsPool] + self._datasets = {} # type: dict[str, ZfsContainer | ZfsBookmark | ZfsSnapshot] self._progress_total_bytes = 0 self._progress_start_time = time.time() ExecuteNode.__init__(self, ssh_config=ssh_config, ssh_to=ssh_to, readonly=readonly, debug_output=debug_output) - def thin(self, objects, keep_objects): + def thin_list(self, snapshots, keep_snapshots): + """Returns a list of snapshots to keep and remove, according to current time and thinner settings. + + :return: ( keeps, removes ) + :type snapshots: list[ZfsSnapshot] + :type keep_snapshots: list[ZfsSnapshot] + :rtype: ( list[ZfsSnapshot], list[ZfsSnapshot] ) + """ # NOTE: if thinning is disabled with --no-thinning, self.__thinner will be none. - if self.__thinner is not None: + if self._thinner is not None: - return self.__thinner.thin(objects, keep_objects, datetime_now(self.utc).timestamp()) + return self._thinner.thin(snapshots, keep_snapshots, datetime_now(self.utc).timestamp()) else: - return (keep_objects, []) + return (keep_snapshots, []) - @CachedProperty + @property def supported_send_options(self): """list of supported options, for optimizing sends""" # not every zfs implementation supports them all - ret = [] - for option in ["-L", "-e", "-c"]: - if self.valid_command(["zfs", "send", option, "zfs_autobackup_option_test"]): - ret.append(option) - return ret + if self._supported_send_options is None: + self._supported_send_options = [] + for option in ["-L", "-e", "-c"]: + if self.valid_command(["zfs", "send", option, "zfs_autobackup_option_test"]): + self._supported_send_options.append(option) + return self._supported_send_options - @CachedProperty + @property def supported_recv_options(self): """list of supported options""" # not every zfs implementation supports them all - ret = [] - for option in ["-s"]: - if self.valid_command(["zfs", "recv", option, "zfs_autobackup_option_test"]): - ret.append(option) - return ret + if self._supported_recv_options is None: + self._supported_recv_options = [] + for option in ["-s"]: + if self.valid_command(["zfs", "recv", option, "zfs_autobackup_option_test"]): + self._supported_recv_options.append(option) + + return self._supported_recv_options def valid_command(self, cmd): """test if a specified zfs options are valid exit code. use this to determine support options""" @@ -108,12 +127,67 @@ class ZfsNode(ExecuteNode): zpool_name = dataset.name.split("/")[0] - return self.__pools.setdefault(zpool_name, ZfsPool(self, zpool_name)) + return self._pools.setdefault(zpool_name, ZfsPool(self, zpool_name)) def get_dataset(self, name, force_exists=None): - """get a ZfsDataset() object from name. stores objects internally to enable caching""" + """get a ZfsDataset() object from name. stores objects internally to enable caching + :type name: str + :rtype: ZfsContainer | ZfsBookmark | ZfsSnapshot + """ - return self.__datasets.setdefault(name, ZfsDataset(self, name, force_exists)) + if name in self._datasets: + return self._datasets[name] + + if '@' in name: + + self._datasets[name] = ZfsSnapshot(self, name, force_exists=force_exists) + elif '#' in name: + + self._datasets[name] = ZfsBookmark(self, name, force_exists=force_exists) + else: + + self._datasets[name] = ZfsContainer(self, name, force_exists=force_exists) + + return self._datasets[name] + + def get_container(self, name, force_exists=None): + """get a ZfsContainer from name, asserting the name contains no @ or #. + :type name: str + :rtype: ZfsContainer + """ + dataset = self.get_dataset(name, force_exists=force_exists) + assert isinstance(dataset, ZfsContainer) + return dataset + + def get_snapshot(self, name, force_exists=None): + """get a ZfsSnapshot from name, asserting the name contains @. + :type name: str + :rtype: ZfsSnapshot + """ + dataset = self.get_dataset(name, force_exists=force_exists) + assert isinstance(dataset, ZfsSnapshot) + return dataset + + def get_bookmark(self, name, force_exists=None): + """get a ZfsBookmark from name, asserting the name contains #. + :type name: str + :rtype: ZfsBookmark + """ + dataset = self.get_dataset(name, force_exists=force_exists) + assert isinstance(dataset, ZfsBookmark) + return dataset + + def get_datasets(self, names, force_exists=None): + """get a list of ZfsDataset() object from names. stores objects internally to enable caching + :rtype: list[ZfsContainer | ZfsBookmark | ZfsSnapshot] + + """ + + ret = [] + for name in names: + ret.append(self.get_dataset(name, force_exists)) + + return ret # def reset_progress(self): # """reset progress output counters""" @@ -149,14 +223,15 @@ class ZfsNode(ExecuteNode): self._progress_start_time = time.time() elif progress_fields[1].isnumeric(): bytes_ = int(progress_fields[1]) - if self._progress_total_bytes: + elapsed = time.time() - self._progress_start_time + if self._progress_total_bytes and elapsed > 0 and bytes_ > 0: percentage = min(100, int(bytes_ * 100 / self._progress_total_bytes)) - speed = int(bytes_ / (time.time() - self._progress_start_time) / (1024 * 1024)) + speed = int(bytes_ / elapsed / (1024 * 1024)) bytes_left = self._progress_total_bytes - bytes_ - minutes_left = int((bytes_left / (bytes_ / (time.time() - self._progress_start_time))) / 60) + minutes_left = int(bytes_left / (bytes_ / elapsed) / 60) self.logger.progress( - "Transfer {}% {}MB/s (total {}MB, {} minutes left)".format(percentage, speed, int( + "Transfering {}% {}MB/s (total {}MB, {} minutes left)".format(percentage, speed, int( self._progress_total_bytes / (1024 * 1024)), minutes_left)) return @@ -188,19 +263,25 @@ class ZfsNode(ExecuteNode): def consistent_snapshot(self, datasets, snapshot_name, min_changed_bytes, pre_snapshot_cmds=[], post_snapshot_cmds=[], set_snapshot_properties=[]): """create a consistent (atomic) snapshot of specified datasets, per pool. + Args: + :type datasets: list[ZfsContainer] """ - pools = {} + pools = {} # type: dict[str, list[ZfsSnapshot]] # collect snapshots that we want to make, per pool # self.debug(datasets) for dataset in datasets: if not dataset.is_changed_ours(min_changed_bytes): - dataset.verbose("No changes since {}".format(dataset.our_snapshots[-1].snapshot_name)) + dataset.verbose("No changes since {}".format(dataset.our_snapshots[-1].suffix)) continue # force_exist, since we're making it - snapshot = self.get_dataset(dataset.name + "@" + snapshot_name, force_exists=True) + snapshot = self.get_snapshot(dataset.name + "@" + snapshot_name, force_exists=True) + + if self.readonly: + snapshot.simulate_properties() + pool = dataset.split_path()[0] if pool not in pools: @@ -208,9 +289,7 @@ class ZfsNode(ExecuteNode): pools[pool].append(snapshot) - # update cache, but try to prevent an unneeded zfs list - if self.readonly or CachedProperty.is_cached(dataset, 'snapshots'): - dataset.snapshots.append(snapshot) # NOTE: this will trigger zfs list if its not cached + dataset.cache_snapshot_bookmark(snapshot, force=self.readonly) if not pools: self.verbose("No changes anywhere: not creating snapshots.") @@ -227,7 +306,7 @@ class ZfsNode(ExecuteNode): for snapshot_property in set_snapshot_properties: cmd += ['-o', snapshot_property] - cmd.extend(map(lambda snapshot_: str(snapshot_), snapshots)) + cmd.extend(map(lambda snapshot_: snapshot_.name, snapshots)) self.verbose("Creating snapshots {} in pool {}".format(snapshot_name, pool_name)) self.run(cmd, readonly=False) @@ -240,22 +319,21 @@ class ZfsNode(ExecuteNode): except Exception as e: pass - def selected_datasets(self, property_name, exclude_received, exclude_paths, exclude_unchanged): + def selected_datasets(self, property_name, exclude_paths, exclude_unchanged): """determine filesystems that should be backed up by looking at the special autobackup-property, systemwide - returns: (list of selected ZfsDataset sorted by createtxg, list of excluded ZfsDataset) + returns: ( list of selected ZfsContainer, list of excluded ZfsContainer) """ self.debug("Getting selected datasets") # get all source filesystems that have the backup property lines = self.run(tab_split=True, readonly=True, cmd=[ - "zfs", "get", "-t", "volume,filesystem", "-Hp", - property_name + ",createtxg" + "zfs", "get", "-t", "volume,filesystem", "-o", "name,value,source", "-H", + property_name ]) - - # The returnlist of selected ZfsDataset's: + # The returnlist of selected ZfsContainer's: selected_filesystems = [] excluded_filesystems = [] @@ -263,14 +341,8 @@ class ZfsNode(ExecuteNode): sources = {} for line in lines: - (name, prop_name, value, raw_source) = line - if prop_name == "createtxg": - # createtxg for the last dataset - if selected_filesystems and selected_filesystems[-1].name == name: - selected_filesystems[-1].createtxg = int(value) - if prop_name != property_name: - continue - dataset = self.get_dataset(name, force_exists=True) + (name, value, raw_source) = line + dataset = self.get_container(name, force_exists=True) # "resolve" inherited sources sources[name] = raw_source @@ -283,14 +355,38 @@ class ZfsNode(ExecuteNode): source = raw_source # determine it - selected=dataset.is_selected(value=value, source=source, inherited=inherited, exclude_received=exclude_received, - exclude_paths=exclude_paths, exclude_unchanged=exclude_unchanged) + selected = dataset.is_selected(value=value, source=source, inherited=inherited, + exclude_paths=exclude_paths, exclude_unchanged=exclude_unchanged) - if selected==True: + if selected == True: selected_filesystems.append(dataset) - elif selected==False: + elif selected == False: excluded_filesystems.append(dataset) - #returns None when no property is set. + # returns None when no property is set. - return (sorted(selected_filesystems, key=attrgetter("createtxg")), - excluded_filesystems) + return (selected_filesystems, excluded_filesystems) + + def get_resume_snapshot(self, target_resume_token): + """returns snapshot that will be resumed by this resume token (run this + on source node with target-token) + + Args: + :type target_resume_token: str + :rtype: ZfsSnapshot|None + """ + # use zfs send -n option to determine this + # NOTE: on smartos stderr, on linux stdout + (stdout, stderr) = self.run(["zfs", "send", "-t", target_resume_token, "-n", "-v"], valid_exitcodes=[0, 255], + readonly=True, return_stderr=True) + if stdout: + lines = stdout + else: + lines = stderr + for line in lines: + matches = re.findall("toname = (.*@.*)", line) + if matches: + snapshot = self.get_snapshot(matches[0]) + snapshot.debug("resume token belongs to this snapshot") + return snapshot + + return None diff --git a/zfs_autobackup/ZfsPointInTime.py b/zfs_autobackup/ZfsPointInTime.py new file mode 100644 index 0000000..43e5985 --- /dev/null +++ b/zfs_autobackup/ZfsPointInTime.py @@ -0,0 +1,103 @@ +import sys + +from .ZfsDataset import ZfsDataset +from datetime import datetime +import time + + +class ZfsPointInTime(ZfsDataset): + """contains stuff that is common to snapshots and bookmarks""" + + + def __init__(self, zfs_node, name, force_exists=None): + + + if type(self) is ZfsPointInTime: + raise TypeError( + "should not be instantiated directly.") + + super().__init__(zfs_node, name, force_exists) + + + #implemented in subclass + @property + def suffix(self): + return "" + + #implemented in subclass + @property + def prefix(self): + return "" + + + @property + def parent(self): + """get parent dataset + + :rtype: ZfsContainer + """ + return self.zfs_node.get_container(self.prefix) + + + @property + def timestamp(self): + """get timestamp from snapshot/bookmark name. Only works for our own snapshots + with the correct format. Snapshots that are not ours always return None + + Note that the tag-part in the name is ignored, so snapsnots are ours regardless of their tag. + + :rtype: int|None + """ + + #note: cant use self.tagless_suffix here, because it depends on us + if self.zfs_node.tag_seperator in self.suffix: + tagless_suffix = self.suffix.split(self.zfs_node.tag_seperator)[0] + else: + tagless_suffix = self.suffix + + try: + dt = datetime.strptime(tagless_suffix, self.zfs_node.snapshot_time_format) + except ValueError: + return None + + if sys.version_info[0] >= 3: + from datetime import timezone + if self.zfs_node.utc: + dt = dt.replace(tzinfo=timezone.utc) + seconds = dt.timestamp() + else: + # python2 has no good functions to deal with UTC. Yet the unix timestamp + # must be in UTC to allow comparison against `time.time()` in on other parts + # of this project (e.g. Thinner.py). If we are handling UTC timestamps, + # we must adjust for that here. + if self.zfs_node.utc: + seconds = (dt - datetime(1970, 1, 1)).total_seconds() + else: + seconds = time.mktime(dt.timetuple()) + return seconds + + @property + def is_ours(self): + """return true if this snapshot name belong to the current backup_name and snapshot formatting""" + return self.timestamp is not None + + + @property + def tagless_suffix(self): + """snapshot or bookmark part of the name, but without the tag. (if its our snapshot we remove the tag)""" + + if self.is_ours and self.zfs_node.tag_seperator in self.suffix: + return self.suffix.split(self.zfs_node.tag_seperator)[0] + else: + return self.suffix + + @property + def tag(self): + """the tag-part of a snapshot or bookmark, if any. can return None""" + + if self.is_ours and self.zfs_node.tag_seperator in self.suffix: + return self.suffix.split(self.zfs_node.tag_seperator)[1] + else: + return None + + diff --git a/zfs_autobackup/ZfsPool.py b/zfs_autobackup/ZfsPool.py index 11976ab..0bc3485 100644 --- a/zfs_autobackup/ZfsPool.py +++ b/zfs_autobackup/ZfsPool.py @@ -1,4 +1,4 @@ -from .CachedProperty import CachedProperty +from logging import warning class ZfsPool(): @@ -10,6 +10,10 @@ class ZfsPool(): self.zfs_node = zfs_node self.name = name + self._properties = None + + def invalidate_cache(self): + self._properties = None def __repr__(self): return "{}: {}".format(self.zfs_node, self.name) @@ -26,28 +30,33 @@ class ZfsPool(): def verbose(self, txt): self.zfs_node.verbose("zpool {}: {}".format(self.name, txt)) + def warning(self, txt): + self.zfs_node.warning("zpool {}: {}".format(self.name, txt)) + def error(self, txt): self.zfs_node.error("zpool {}: {}".format(self.name, txt)) def debug(self, txt): self.zfs_node.debug("zpool {}: {}".format(self.name, txt)) - @CachedProperty + @property def properties(self): """all zpool properties""" - self.debug("Getting zpool properties") + if self._properties is None: - cmd = [ - "zpool", "get", "-H", "-p", "all", self.name - ] + self.debug("Getting zpool properties") - ret = {} + cmd = [ + "zpool", "get", "-H", "-p", "all", self.name + ] - for pair in self.zfs_node.run(tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]): - ret[pair[1]] = pair[2] + self._properties = {} - return ret + for pair in self.zfs_node.run(tab_split=True, cmd=cmd, readonly=True, valid_exitcodes=[0]): + self._properties[pair[1]] = pair[2] + + return self._properties @property def features(self): @@ -60,4 +69,4 @@ class ZfsPool(): if value == 'enabled' or value == 'active': ret.append(feature) - return ret \ No newline at end of file + return ret diff --git a/zfs_autobackup/ZfsSnapshot.py b/zfs_autobackup/ZfsSnapshot.py new file mode 100644 index 0000000..df1e40c --- /dev/null +++ b/zfs_autobackup/ZfsSnapshot.py @@ -0,0 +1,330 @@ +from typing import cast + +from .ZfsPointInTime import ZfsPointInTime + + +class ZfsSnapshot(ZfsPointInTime): + """A ZFS snapshot""" + + @property + def prefix(self): + (filesystem, snapshot) = self.name.split("@") + return filesystem + + @property + def suffix(self): + (filesystem, snapshot_name) = self.name.split("@") + return snapshot_name + + @property + def typed_suffix(self): + return "@" + self.suffix + + + @property + def is_snapshot_excluded(self): + """true if this is a snapshot and matches the exclude pattern""" + suffix = self.suffix + for pattern in self.zfs_node.exclude_snapshot_patterns: + if pattern.search(suffix) is not None: + self.debug("Excluded (path matches snapshot exclude pattern)") + return True + return False + + def find_snapshot_by_suffix(self, snapshot_bookmarks): + """return ZfsSnapshot|ZfsBookmark from the list of snapshot_bookmarks, if it matches our snapshot_name. Otherwise None + Args: + :type snapshot_bookmarks: list[ZfsSnapshot|ZfsBookmark] + :rtype: ZfsSnapshot|ZfsBookmark|None + """ + + for snapshot_bookmark in snapshot_bookmarks: + if snapshot_bookmark.tagless_suffix == self.tagless_suffix: + return snapshot_bookmark + + return None + + def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, write_embedded, + send_pipes, zfs_compressed): + """returns a pipe with zfs send output for this snapshot + + resume_token: resume sending from this token. (in that case we don't + need to know snapshot names) + + If prev_snapshot is on a different parent dataset (clone-of-origin case), + its full name is passed to zfs send -i; otherwise the short @suffix form is used. + + Args: + :param send_pipes: output cmd array that will be added to actual zfs send command. (e.g. mbuffer or compression program) + :type send_pipes: list[str] + :type features: list[str] + :type prev_snapshot: ZfsSnapshot|ZfsBookmark|None + :type resume_token: str|None + :type show_progress: bool + :type raw: bool + :type send_properties: bool + :type write_embedded: bool + :type zfs_compressed: bool + """ + # build source command + cmd = [] + + cmd.extend(["zfs", "send", ]) + + # all kind of performance options: + if 'large_blocks' in features and "-L" in self.zfs_node.supported_send_options: + # large block support (only if recordsize>128k which is seldomly used) + cmd.append("-L") # --large-block + + if write_embedded and 'embedded_data' in features and "-e" in self.zfs_node.supported_send_options: + cmd.append("-e") # --embed; WRITE_EMBEDDED, more compact stream + + if zfs_compressed and "-c" in self.zfs_node.supported_send_options: + cmd.append("-c") # --compressed; use compressed WRITE records + + # raw? (send over encrypted data in its original encrypted form without decrypting) + if raw: + cmd.append("--raw") + + # progress output + if show_progress: + cmd.append("-v") # --verbose + cmd.append("-P") # --parsable + + # resume a previous send? (don't need more parameters in that case) + if resume_token: + cmd.extend(["-t", resume_token]) + + else: + # send properties + if send_properties: + cmd.append("-p") # --props + + # incremental? + if prev_snapshot: + if prev_snapshot.prefix != self.prefix: + # cross-dataset (clone replicated from its origin snapshot): full path required + cmd.extend(["-i", prev_snapshot.name]) + else: + cmd.extend(["-i", prev_snapshot.typed_suffix]) + + cmd.append(self.name) + + cmd.extend(send_pipes) + + output_pipe = self.zfs_node.run(cmd, pipe=True, readonly=True) + + return output_pipe + + def recv_pipe(self, pipe, features, recv_pipes, filter_properties=None, set_properties=None, ignore_exit_code=False, + force=False): + """starts a zfs recv for this snapshot and uses pipe as input + + note: you can it both on a snapshot or filesystem object. The + resulting zfs command is the same, only our object cache is invalidated + differently. + + Args: + :param recv_pipes: input cmd array that will be prepended to actual zfs recv command. (e.g. mbuffer or decompression program) + :type pipe: subprocess.pOpen + :type features: list[str] + :type filter_properties: list[str] + :type set_properties: list[str] + :type ignore_exit_code: bool + :type force: bool + """ + + if set_properties is None: + set_properties = [] + + if filter_properties is None: + filter_properties = [] + + # build target command + cmd = [] + + cmd.extend(recv_pipes) + + cmd.extend(["zfs", "recv"]) + + # don't let zfs recv mount everything thats received (even with canmount=noauto!) + cmd.append("-u") + + for property_ in filter_properties: + cmd.extend(["-x", property_]) + + for property_ in set_properties: + cmd.extend(["-o", property_]) + + # verbose output + cmd.append("-v") + + if force: + cmd.append("-F") + + if 'extensible_dataset' in features and "-s" in self.zfs_node.supported_recv_options: + # support resuming + self.debug("Enabled resume support") + cmd.append("-s") + + cmd.append(self.prefix) + + if ignore_exit_code: + valid_exitcodes = [] + else: + valid_exitcodes = [0] + + # self.zfs_node.reset_progress() + self.zfs_node.run(cmd, inp=pipe, valid_exitcodes=valid_exitcodes) + + # invalidate cache + self.invalidate_cache() + + # in test mode we pretend everything was ok and it exists + if self.zfs_node.readonly: + self.force_exists = True + self.simulate_properties() + + # check if transfer was really ok (exit codes have been wrong before due to bugs in zfs-utils and some + # errors should be ignored, thats where the ignore_exitcodes is for.) + if not self.exists: + self.error("error during transfer") + raise (Exception("Target doesn't exist after transfer, something went wrong.")) + + # at this point we're sure the actual dataset exists + self.parent.force_exists = True + + def transfer_snapshot(self, target_snapshot, features, prev_snapshot, show_progress, + filter_properties, set_properties, ignore_recv_exit_code, resume_token, + raw, send_properties, write_embedded, send_pipes, recv_pipes, zfs_compressed, force): + """transfer this snapshot to target_snapshot. specify prev_snapshot for + incremental transfer + + connects a send_pipe() to recv_pipe() + + Args: + :type send_pipes: list[str] + :type recv_pipes: list[str] + :type target_snapshot: ZfsSnapshot + :type features: list[str] + :type prev_snapshot: ZfsSnapshot|ZfsBookmark|None + :type show_progress: bool + :type filter_properties: list[str] + :type set_properties: list[str] + :type ignore_recv_exit_code: bool + :type resume_token: str|None + :type raw: bool + :type send_properties: bool + :type write_embedded: bool + :type zfs_compressed: bool + :type force: bool + """ + + if set_properties is None: + set_properties = [] + if filter_properties is None: + filter_properties = [] + + self.debug("Transfer snapshot to {}".format(target_snapshot.prefix)) + + if resume_token: + self.verbose("resuming") + + # initial or increment + if not prev_snapshot: + self.verbose("-> {} (new)".format(target_snapshot.prefix)) + else: + # incremental + self.verbose("-> {}".format(target_snapshot.prefix)) + + if show_progress: + self.zfs_node.logger.progress("Transferring...") + + # do it + pipe = self.send_pipe(features=features, show_progress=show_progress, prev_snapshot=prev_snapshot, + resume_token=resume_token, raw=raw, send_properties=send_properties, + write_embedded=write_embedded, send_pipes=send_pipes, zfs_compressed=zfs_compressed) + target_snapshot.recv_pipe(pipe, features=features, filter_properties=filter_properties, + set_properties=set_properties, ignore_exit_code=ignore_recv_exit_code, + recv_pipes=recv_pipes, force=force) + + # try to automount it, if its the initial transfer + if not prev_snapshot: + # in test mode it doesnt actually exist, so dont try to mount it/read properties, and simulate properties. + if not target_snapshot.zfs_node.readonly: + target_snapshot.parent.automount() + + def bookmark(self, tag): + """Bookmark this snapshot, and return the bookmark. If the bookmark already exist, it returns it.""" + # NOTE: we use the tag to add the target_path GUID, so that we can have multiple bookmarks for the same snapshot, but for different targets. + # This is to make sure you can send a backup to two locations, without them interfering with eachothers bookmarks. + + + bookmark_name = self.prefix + "#" + self.tagless_suffix + self.zfs_node.tag_seperator + tag + + # does this exact bookmark (including tag) already exists? + existing_bookmark = self.parent.find_exact_bookmark(bookmark_name) + if existing_bookmark is not None: + return existing_bookmark + + self.debug("Bookmarking {}".format(bookmark_name)) + + cmd = [ + "zfs", "bookmark", self.name, bookmark_name + ] + + self.zfs_node.run(cmd=cmd) + + bookmark = self.zfs_node.get_bookmark(bookmark_name, force_exists=True) + self.parent.cache_snapshot_bookmark(bookmark) + return bookmark + + @property + def holds(self): + + output = cast("list[list[str]]", cast(object, self.zfs_node.run( + ["zfs", "holds", "-H", self.name], valid_exitcodes=[0], tab_split=True, readonly=True))) + return map(lambda fields: fields[1], output) + + def is_hold(self): + """did we hold this snapshot?""" + return self.zfs_node.hold_name in self.holds + + def hold(self): + """hold snapshot""" + self.debug("holding") + self.zfs_node.run(["zfs", "hold", self.zfs_node.hold_name, self.name], valid_exitcodes=[0, 1]) + + def release(self): + """release snapshot""" + if self.zfs_node.readonly or self.is_hold(): + self.debug("releasing") + self.zfs_node.run(["zfs", "release", self.zfs_node.hold_name, self.name], valid_exitcodes=[0, 1]) + + + def rollback(self): + + self.debug("Rolling back") + self.zfs_node.run(["zfs", "rollback", self.name]) + + def destroy(self, fail_exception=False, deferred=False): + + self.verbose("Destroying") + self.release() + return super().destroy(fail_exception=fail_exception, deferred=deferred) + + def clone(self, name): + """clones this snapshot and returns ZfsContainer of the clone + :rtype: ZfsContainer + """ + + self.debug("Cloning to {}".format(name)) + + cmd = [ + "zfs", "clone", self.name, name + ] + + self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) + + return self.zfs_node.get_container(name, force_exists=True) + diff --git a/zfs_autobackup/compressors.py b/zfs_autobackup/compressors.py index 40b39e5..b1ce8ec 100644 --- a/zfs_autobackup/compressors.py +++ b/zfs_autobackup/compressors.py @@ -7,69 +7,72 @@ COMPRESS_CMDS = { 'gzip': { 'cmd': 'gzip', - 'args': [ '-3' ], + 'args': ['-3'], 'dcmd': 'zcat', 'dargs': [], }, 'pigz-fast': { 'cmd': 'pigz', - 'args': [ '-3' ], + 'args': ['-3'], 'dcmd': 'pigz', - 'dargs': [ '-dc' ], + 'dargs': ['-dc'], }, 'pigz-slow': { 'cmd': 'pigz', - 'args': [ '-9' ], + 'args': ['-9'], 'dcmd': 'pigz', - 'dargs': [ '-dc' ], + 'dargs': ['-dc'], }, 'zstd-fast': { 'cmd': 'zstdmt', - 'args': [ '-3' ], + 'args': ['-3'], 'dcmd': 'zstdmt', - 'dargs': [ '-dc' ], + 'dargs': ['-dc'], }, 'zstd-slow': { 'cmd': 'zstdmt', - 'args': [ '-19' ], + 'args': ['-19'], 'dcmd': 'zstdmt', - 'dargs': [ '-dc' ], + 'dargs': ['-dc'], }, 'zstd-adapt': { 'cmd': 'zstdmt', - 'args': [ '--adapt' ], + 'args': ['--adapt'], 'dcmd': 'zstdmt', - 'dargs': [ '-dc' ], + 'dargs': ['-dc'], }, 'xz': { 'cmd': 'xz', 'args': [], 'dcmd': 'xz', - 'dargs': [ '-d' ], + 'dargs': ['-d'], }, 'lzo': { 'cmd': 'lzop', 'args': [], 'dcmd': 'lzop', - 'dargs': [ '-dfc' ], + 'dargs': ['-dfc'], }, 'lz4': { 'cmd': 'lz4', 'args': [], 'dcmd': 'lz4', - 'dargs': [ '-dc' ], + 'dargs': ['-dc'], }, } + def compress_cmd(compressor): - ret=[ COMPRESS_CMDS[compressor]['cmd'] ] - ret.extend( COMPRESS_CMDS[compressor]['args']) + ret = [COMPRESS_CMDS[compressor]['cmd']] + ret.extend(COMPRESS_CMDS[compressor]['args']) return ret + def decompress_cmd(compressor): - ret= [ COMPRESS_CMDS[compressor]['dcmd'] ] + ret = [COMPRESS_CMDS[compressor]['dcmd']] ret.extend(COMPRESS_CMDS[compressor]['dargs']) return ret + def choices(): return COMPRESS_CMDS.keys() diff --git a/zfs_autobackup/sync_datasets.py b/zfs_autobackup/sync_datasets.py new file mode 100644 index 0000000..227f243 --- /dev/null +++ b/zfs_autobackup/sync_datasets.py @@ -0,0 +1,297 @@ +"""Dataset-level replication: sync a selection of source datasets to a target node, +and thin or destroy target datasets that are missing on the source.""" + +from collections import deque + +from .ThinnerRule import ThinnerRule +from .ZfsContainer import ZfsContainer +from .sync_snapshots import sync_snapshots +from .util import datetime_now + + +def _missing_datasets(target_dataset, used_target_datasets): + """list the datasets under target_dataset that are not used as a backup target (anymore) + + :type target_dataset: ZfsContainer + :type used_target_datasets: list[ZfsContainer] + :rtype: list[ZfsContainer] + """ + + return [dataset for dataset in target_dataset.recursive_datasets if dataset not in used_target_datasets] + + +def thin_missing_targets(logger, target_dataset, used_target_datasets): + """thin target datasets that are missing on the source. + + :type logger: LogConsole + :type target_dataset: ZfsContainer + :type used_target_datasets: list[ZfsContainer] + """ + + logger.debug("Thinning obsolete datasets") + missing_datasets = _missing_datasets(target_dataset, used_target_datasets) + + for count, dataset in enumerate(missing_datasets, start=1): + logger.debug("analyse missing {}".format(dataset)) + logger.progress("Analysing missing..", count, len(missing_datasets), 0) + + try: + dataset.debug("Missing on source, thinning") + dataset.thin() + except Exception as e: + dataset.error("Error during thinning of missing datasets ({})".format(str(e))) + + +def _destroy_missing_dataset(dataset, destroy_missing, utc): + """destroy a single missing dataset if it meets all the requirements, otherwise tell the user why not. + + :type dataset: ZfsContainer + :type destroy_missing: str + :type utc: bool + """ + + # cant do anything without our own snapshots + if not dataset.our_snapshots: + if dataset.datasets: + # its not a leaf, just ignore + dataset.debug("Destroy missing: ignoring") + else: + dataset.verbose("Destroy missing: has no snapshots made by us (please destroy manually).") + return + + # past the deadline? + deadline_ttl = ThinnerRule("0s" + destroy_missing).ttl + now = datetime_now(utc).timestamp() + if dataset.our_snapshots[-1].timestamp + deadline_ttl > now: + dataset.verbose("Destroy missing: Waiting for deadline.") + return + + dataset.debug("Destroy missing: Removing our snapshots.") + + # remove all our snaphots, except last, to safe space in case we fail later on + for snapshot in dataset.our_snapshots[:-1]: + snapshot.destroy(fail_exception=True) + + # does it have other snapshots? + if any(not snapshot.is_ours for snapshot in dataset.snapshots): + dataset.verbose("Destroy missing: Still in use by other snapshots") + return + + if dataset.datasets: + dataset.verbose("Destroy missing: Still has children here.") + return + + dataset.verbose("Destroy missing.") + dataset.our_snapshots[-1].destroy(fail_exception=True) + dataset.destroy(fail_exception=True) + + +def destroy_missing_targets(logger, target_dataset, used_target_datasets, destroy_missing, utc): + """destroy target datasets that are missing on the source and that meet the requirements + + :type logger: LogConsole + :type target_dataset: ZfsContainer + :type used_target_datasets: list[ZfsContainer] + :type destroy_missing: str + :type utc: bool + """ + + logger.debug("Destroying obsolete datasets") + missing_datasets = _missing_datasets(target_dataset, used_target_datasets) + + for count, dataset in enumerate(missing_datasets, start=1): + logger.progress("Analysing destroy missing...", count, len(missing_datasets), 0) + + try: + _destroy_missing_dataset(dataset, destroy_missing, utc) + except Exception as e: + dataset.error("Error during --destroy-missing: {}".format(str(e))) + + +def _clone_dependencies(source_datasets): + """yield (clone_dataset, origin_snapshot, origin_dataset) for every selected dataset that is a + clone of another selected dataset. + + :type source_datasets: list[ZfsContainer] + """ + + for dataset in source_datasets: + origin_snapshot = dataset.origin + if origin_snapshot is None: + continue + origin_dataset = origin_snapshot.parent + # only a dependency if the origin dataset is itself in the selection + if origin_dataset in source_datasets and origin_dataset is not dataset: + yield dataset, origin_snapshot, origin_dataset + + +def _topological_sort_for_clones(logger, source_datasets): + """Reorder source_datasets so an origin is processed before any selected clone of it. + Datasets whose origin is not in the selection (or who are not clones) are independent + and keep their relative order. On a detected cycle (should not happen with valid ZFS) + the original list is returned with a warning. + + :type logger: LogConsole + :type source_datasets: list[ZfsContainer] + :rtype: list[ZfsContainer] + """ + + # build edge map: for each dataset that is a clone of another selected dataset, + # record that dependency so the origin is emitted first. + indegree = {dataset: 0 for dataset in source_datasets} + children = {dataset: [] for dataset in source_datasets} + for clone_dataset, origin_snapshot, origin_dataset in _clone_dependencies(source_datasets): + indegree[clone_dataset] += 1 + children[origin_dataset].append(clone_dataset) + + # Kahn's algorithm: start with datasets that have no in-selection origin + queue = deque(dataset for dataset in source_datasets if indegree[dataset] == 0) + result = [] + while queue: + dataset = queue.popleft() + result.append(dataset) + for child in children[dataset]: + indegree[child] -= 1 + if indegree[child] == 0: + queue.append(child) + + # a cycle means ZFS state is inconsistent; fall back to original order + if len(result) != len(source_datasets): + logger.warning("Clone topological sort: cycle detected, keeping original dataset order.") + return source_datasets + + return result + + +def _required_origin_snapshots(source_datasets): + """Build a map origin_dataset -> set of origin snapshots that downstream clones (also in the + selection) need pinned on the target. Used to force-include those specific snapshots even + without --other-snapshots. + + :type source_datasets: list[ZfsContainer] + :rtype: dict[ZfsContainer, set[ZfsSnapshot]] + """ + + required = {} + for clone_dataset, origin_snapshot, origin_dataset in _clone_dependencies(source_datasets): + required.setdefault(origin_dataset, set()).add(origin_snapshot) + return required + + +def sync_datasets(logger, source_node, source_datasets, target_node, bookmark_tag, + send_pipes, recv_pipes, target_path, strip_path, + no_clone, no_send, no_bookmarks, no_thinning, + filter_properties, set_properties, + ignore_transfer_errors, holds, rollback, other_snapshots, + destroy_incompatible, decrypt, encrypt, zfs_compressed, force, + guid_check, property_format, debug, + target_dataset_base, destroy_missing, utc): + """Sync datasets, or thin-only on both sides. + + :type logger: LogConsole + :type source_node: ZfsNode + :type source_datasets: list[ZfsContainer] + :type target_node: ZfsNode + :type bookmark_tag: str + :type send_pipes: list[str] + :type recv_pipes: list[str] + :type target_path: str + :type strip_path: int + :type no_clone: bool + :type no_send: bool + :type no_bookmarks: bool + :type no_thinning: bool + :type filter_properties: list[str] + :type set_properties: list[str] + :type ignore_transfer_errors: bool + :type holds: bool + :type rollback: bool + :type other_snapshots: bool + :type destroy_incompatible: bool + :type decrypt: bool + :type encrypt: bool + :type zfs_compressed: bool + :type force: bool + :type guid_check: bool + :type property_format: str + :type debug: bool + :type target_dataset_base: ZfsContainer + :type destroy_missing: str|None + :type utc: bool + :rtype: int + """ + + if not no_clone: + source_datasets = _topological_sort_for_clones(logger, source_datasets) + required_origin_snapshots = _required_origin_snapshots(source_datasets) + else: + required_origin_snapshots = {} + + fail_count = 0 + target_datasets = [] + bookmarks_unsupported_pools = set() # source pools we've already warned about + for count, source_dataset in enumerate(source_datasets, start=1): + + logger.progress("Analysing dataset...", count, len(source_datasets), fail_count) + + try: + # determine corresponding target_dataset + target_name = source_dataset.map_to_target_path(target_path, strip_path) + target_dataset = target_node.get_container(target_name) + target_datasets.append(target_dataset) + + # ensure parents exists + target_parent = target_dataset.parent + if not no_send \ + and target_parent is not None \ + and target_parent not in target_datasets \ + and not target_parent.exists: + target_dataset.debug("Creating unmountable parents") + target_parent.create_filesystem(parents=True) + + # determine common zpool features + source_pool = source_node.get_pool(source_dataset) + source_features = source_pool.features + target_features = target_node.get_pool(target_dataset).features + common_features = [feature for feature in source_features if feature in target_features] + + use_bookmarks = not no_bookmarks + if use_bookmarks and 'bookmarks' not in common_features: + if source_pool.name not in bookmarks_unsupported_pools: + source_pool.warning("Disabling bookmarks, not supported on both pools.") + bookmarks_unsupported_pools.add(source_pool.name) + use_bookmarks = False + + # sync the snapshots of this dataset + sync_snapshots(source_dataset, target_dataset, show_progress=True, + features=common_features, filter_properties=filter_properties, + set_properties=set_properties, + ignore_recv_exit_code=ignore_transfer_errors, + holds=holds, rollback=rollback, + also_other_snapshots=other_snapshots, + no_send=no_send, + destroy_incompatible=destroy_incompatible, + send_pipes=send_pipes, recv_pipes=recv_pipes, + decrypt=decrypt, encrypt=encrypt, + zfs_compressed=zfs_compressed, force=force, + guid_check=guid_check, use_bookmarks=use_bookmarks, + bookmark_tag=bookmark_tag, + property_format=property_format, + no_clone=no_clone, + target_path=target_path, strip_path=strip_path, + required_snapshots=required_origin_snapshots.get(source_dataset)) + except Exception as e: + fail_count = fail_count + 1 + source_dataset.error("FAILED: " + str(e)) + if debug: + logger.verbose("Debug mode, aborting on first error") + raise + + if not no_thinning: + thin_missing_targets(logger, target_dataset=target_dataset_base, used_target_datasets=target_datasets) + + if destroy_missing is not None: + destroy_missing_targets(logger, target_dataset=target_dataset_base, used_target_datasets=target_datasets, + destroy_missing=destroy_missing, utc=utc) + + return fail_count diff --git a/zfs_autobackup/sync_snapshots.py b/zfs_autobackup/sync_snapshots.py new file mode 100644 index 0000000..49b8eb2 --- /dev/null +++ b/zfs_autobackup/sync_snapshots.py @@ -0,0 +1,483 @@ +"""Snapshot-level replication: sync the snapshots of one dataset to a target dataset, +thinning out obsolete snapshots on both sides along the way.""" + +from .ZfsBookmark import ZfsBookmark +from .ZfsContainer import ZfsContainer +from .ZfsPointInTime import ZfsPointInTime +from .ZfsSnapshot import ZfsSnapshot + + +def _resolve_clone_origin(source_dataset, target_node, target_path, strip_path, guid_check): + """Return the source-side origin snapshot to use as zfs send -i base for a clone, + or None if the clone relationship cannot be preserved on the target. + + :type source_dataset: ZfsContainer + :type target_node: ZfsNode + :type target_path: str + :type strip_path: int + :type guid_check: bool + :rtype: ZfsSnapshot|None + """ + + source_origin_snap = source_dataset.origin + if source_origin_snap is None: + return None + + # Reverse-clone topology (after 'zfs promote' of a child): origin lives on a namespace descendant + # of this dataset. zfs recv cannot land a clone-creating stream on top of the placeholder that has + # to exist for the descendant, so replication can't preserve the relationship. + origin_parent_path = source_origin_snap.name.split('@', 1)[0] + if origin_parent_path == source_dataset.name or origin_parent_path.startswith(source_dataset.name + "/"): + source_dataset.warning( + "Cannot replicate as clone: origin '{}' lives on a namespace descendant of this dataset. " + "Falling back to full send.".format(source_origin_snap.name)) + return None + + try: + target_origin_parent = source_origin_snap.parent.map_to_target_path(target_path, strip_path) + except Exception as e: + source_dataset.warning( + "Cannot replicate as clone: cannot map origin '{}' to target ({}). " + "Falling back to full send.".format(source_origin_snap.name, str(e))) + return None + + target_origin_snap = target_node.get_snapshot(target_origin_parent + '@' + source_origin_snap.suffix) + + if not target_origin_snap.exists: + source_dataset.warning( + "Cannot replicate as clone: origin '{}' not available on target. " + "Falling back to full send.".format(source_origin_snap.name)) + return None + + if guid_check and not source_origin_snap.guid_matches(target_origin_snap): + source_dataset.warning( + "Cannot replicate as clone: origin guid mismatch between source and target. " + "Falling back to full send.") + return None + + return source_origin_snap + + +def _pre_clean(source_dataset, target_dataset, source_common_snapshot, source_obsoletes, target_obsoletes, + target_transfers): + """cleanup old stuff on source and target before starting snapshot syncing + + :type source_dataset: ZfsContainer + :type target_dataset: ZfsContainer + :type source_common_snapshot: ZfsSnapshot|ZfsBookmark|None + :type source_obsoletes: list[ZfsSnapshot] + :type target_obsoletes: list[ZfsSnapshot] + :type target_transfers: list[ZfsSnapshot] + """ + + # on source: delete all obsoletes that are not in target_transfers (except common snapshot, if its not a bookmark) + for source_snapshot in source_dataset.snapshots: + if (source_snapshot in source_obsoletes + and source_common_snapshot != source_snapshot + and source_snapshot.find_snapshot_by_suffix(target_transfers) is None): + source_snapshot.destroy() + + # on target: destroy everything thats obsolete, except the common snapshot + if not target_dataset.exists: + return + + for target_snapshot in target_dataset.snapshots: + if (target_snapshot in target_obsoletes + and (not source_common_snapshot + or target_snapshot.tagless_suffix != source_common_snapshot.tagless_suffix) + and target_snapshot.exists): + target_snapshot.destroy() + + +def _handle_incompatible_target(target_dataset, incompatible_target_snapshots, destroy_incompatible, force, + source_common_snapshot, is_clone_origin, is_resume): + """destroy incompatible snapshots on target before sync, or inform user what to do + + :type target_dataset: ZfsContainer + :type incompatible_target_snapshots: list[ZfsSnapshot] + :type destroy_incompatible: bool + :type force: bool + :type source_common_snapshot: ZfsPointInTime|None + :type is_clone_origin: bool + :type is_resume: bool + """ + + # a clone origin is not a real common snapshot of this dataset: it lives on a different + # target dataset, so it doesnt make existing content of this target dataset compatible. + if is_clone_origin: + source_common_snapshot = None + + if not target_dataset.exists: + # no target yet, so everything ok + return + + if not incompatible_target_snapshots and source_common_snapshot is not None: + # nice existing target with compatible common snapshot. + return + + if not incompatible_target_snapshots and is_resume and len(target_dataset.snapshots) == 0: + # its a dataset from an incremental send we probably can resume, so its fine + return + + # from this point on things get progressively worse.. + + if source_common_snapshot: + # we just need to delete some incompatibles: + for snapshot in incompatible_target_snapshots: + snapshot.warning("Incompatible snapshot") + + if not destroy_incompatible: + raise Exception("Use --destroy-incompatible to get rid of these.") + else: + # no common snapshot, so the whole dataset is incompatible + if incompatible_target_snapshots: + if not force or not destroy_incompatible: + target_dataset.error("Incompatible dataset!") + raise Exception("Use --destroy-incompatible -F to overwrite the target dataset and start over.") + elif not force: + target_dataset.error("Incompatible dataset!") + raise Exception("Use -F to overwrite the target dataset and start over.") + + target_dataset.warning("Overwriting incompatible dataset.") + + # remove incompatibles and rollback + for snapshot in incompatible_target_snapshots: + snapshot.destroy(fail_exception=True) + + target_dataset.rollback() + + +def _validate_resume_token(source_dataset, target_dataset, start_snapshot): + """validate and return the resume token, or abort (destroy) the resume state if its no longer valid + + :type source_dataset: ZfsContainer + :type target_dataset: ZfsContainer + :type start_snapshot: ZfsSnapshot|None + :rtype: str|None + """ + + if not target_dataset.exists or 'receive_resume_token' not in target_dataset.properties: + return None + + if start_snapshot is None: + target_dataset.verbose("Aborting resume, its obsolete.") + target_dataset.abort_resume() + return None + + resume_token = target_dataset.properties['receive_resume_token'] + + # not valid anymore? + resume_snapshot = source_dataset.zfs_node.get_resume_snapshot(resume_token) + if not resume_snapshot or start_snapshot.suffix != resume_snapshot.suffix: + target_dataset.verbose("Aborting resume, its no longer valid.") + target_dataset.abort_resume() + return None + + return resume_token + + +def _plan_sync(source_dataset, target_dataset, also_other_snapshots, guid_check, raw, bookmark_tag, + no_clone, target_path, strip_path, required_snapshots=None): + """Determine at what snapshot to start syncing to target_dataset and what to sync and what to keep. + + :type source_dataset: ZfsContainer + :type target_dataset: ZfsContainer + :type also_other_snapshots: bool + :type guid_check: bool + :type raw: bool + :type bookmark_tag: str + :type no_clone: bool + :type target_path: str + :type strip_path: int + :type required_snapshots: set[ZfsSnapshot]|None + :rtype: ( ZfsSnapshot|ZfsBookmark|None, bool, list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot], list[ZfsSnapshot] ) + + Returns: + tuple: A tuple containing: + - ZfsSnapshot|ZfsBookmark|None: The source common snapshot. (the clone origin snapshot when is_clone_origin is True) + - bool: is_clone_origin: the common snapshot is actually the clone origin, which lives on a different dataset. + - list[ZfsSnapshot]: Our obsolete source snapshots, after transfer is done. (will be thinned asap) + - list[ZfsSnapshot]: Our obsolete target snapshots, after transfer is done. (will be thinned asap) + - list[ZfsSnapshot]: Transfer target snapshots. These need to be transferred. + - list[ZfsSnapshot]: Incompatible target snapshots. Target snapshots that are in the way, after the common snapshot. (need to be destroyed to continue) + """ + + ### 1: determine common and start snapshot if target already exists: + if target_dataset.exists: + target_dataset.debug("Determining start snapshot") + (source_common_snapshot, target_common_snapshot) = source_dataset.find_common_snapshot( + target_dataset, guid_check=guid_check, bookmark_tag=bookmark_tag) + incompatible_target_snapshots = target_dataset.find_incompatible_snapshots(target_common_snapshot, raw) + else: + source_common_snapshot = None + incompatible_target_snapshots = [] + + # let thinner decide whats obsolete on source after the transfer is done + if source_dataset.our_snapshots: + source_obsoletes = source_dataset.thin_list()[1] + else: + source_obsoletes = [] + + ### 2: determine possible target snapshots + + # start with snapshots that already exist, minus incompatibles + if target_dataset.exists: + possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots + if snapshot not in incompatible_target_snapshots] + else: + possible_target_snapshots = [] + + # add all snapshots from the source, starting after the common snapshot if it exists + if source_common_snapshot is not None: + source_snapshot = source_dataset.find_next_snapshot(source_common_snapshot) + elif source_dataset.snapshots: + source_snapshot = source_dataset.snapshots[0] + else: + source_snapshot = None + + while source_snapshot: + # we want it? + is_required = required_snapshots is not None and source_snapshot in required_snapshots + if is_required and not source_snapshot.is_ours and not also_other_snapshots: + source_snapshot.verbose("Including as clone origin for a selected clone") + if (also_other_snapshots or source_snapshot.is_ours or is_required) \ + and not source_snapshot.is_snapshot_excluded: + # create virtual target snapshot + target_snapshot = target_dataset.zfs_node.get_snapshot( + target_dataset.name + source_snapshot.typed_suffix, force_exists=False) + possible_target_snapshots.append(target_snapshot) + source_snapshot = source_dataset.find_next_snapshot(source_snapshot) + + ### 3: let the thinner decide what it wants by looking at all the possible target snapshots at once. + # always keep the last target snapshot as common snapshot. + if possible_target_snapshots: + (target_keeps, target_obsoletes) = target_dataset.zfs_node.thin_list( + possible_target_snapshots, keep_snapshots=[possible_target_snapshots[-1]]) + else: + target_keeps = [] + target_obsoletes = [] + + ### 4: look at what the thinner wants to keep and create a list of snapshots we still need to transfer + target_transfers = [target_keep for target_keep in target_keeps if not target_keep.exists] + + ### 5: no common snapshot, but the source is a clone? then the origin acts as the common snapshot: + # the first send becomes an incremental from the origin, so zfs recv reconstructs the clone + # relationship on the target. + is_clone_origin = False + if source_common_snapshot is None and not no_clone and target_transfers: + source_common_snapshot = _resolve_clone_origin(source_dataset, target_dataset.zfs_node, target_path, + strip_path, guid_check) + is_clone_origin = source_common_snapshot is not None + + return (source_common_snapshot, is_clone_origin, source_obsoletes, target_obsoletes, target_transfers, + incompatible_target_snapshots) + + +def _active_properties(source_dataset, filter_properties, set_properties, property_format): + """determine the property filter/set lists to use for this transfer + + :type source_dataset: ZfsContainer + :type filter_properties: list[str] + :type set_properties: list[str] + :type property_format: str + :rtype: (list[str], list[str]) + """ + + (active_filter_properties, active_set_properties) = source_dataset.get_allowed_properties(filter_properties, + set_properties) + + # always filter properties that start with the property-format prefix (https://github.com/psy0rz/zfs_autobackup/issues/221) + property_prefix = property_format.split(':')[0] + for prop in source_dataset.properties: + if prop.startswith(property_prefix): + active_filter_properties.append(prop) + + return active_filter_properties, active_set_properties + + +def _post_transfer_source_cleanup(source_snapshot, prev_base, source_obsoletes, holds, use_bookmarks, bookmark_tag): + """after source_snapshot was transferred: hold or bookmark it as the new common snapshot, and + release/destroy the previous one if its obsolete. Returns the source-side point-in-time to use + as base for the next incremental send. + + :type source_snapshot: ZfsSnapshot + :type prev_base: ZfsPointInTime|None + :type source_obsoletes: list[ZfsSnapshot] + :type holds: bool + :type use_bookmarks: bool + :type bookmark_tag: str + :rtype: ZfsPointInTime + """ + + if use_bookmarks: + # bookmark the new common snapshot, so the snapshot itself can be destroyed when obsolete + source_bookmark = source_snapshot.bookmark(bookmark_tag) + if source_snapshot in source_obsoletes: + source_snapshot.destroy() + + # TODO: make a better is_ours specially for bookmarks, as part of the next refactoring splitting in more classes + # delete the previous bookmark if its ours (ends in our tag_separator + tag) + if isinstance(prev_base, ZfsBookmark) and prev_base.name.endswith( + source_snapshot.zfs_node.tag_seperator + bookmark_tag): + prev_base.destroy() + + return source_bookmark + + # not using bookmarks: hold the new common snapshot instead + if holds: + source_snapshot.hold() + + # release hold and cleanup the previous common snapshot if its obsolete + if isinstance(prev_base, ZfsSnapshot): + prev_base.release() + if prev_base in source_obsoletes: + prev_base.destroy() + + return source_snapshot + + +def sync_snapshots(source_dataset, target_dataset, features, show_progress, filter_properties, set_properties, + ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, + no_send, destroy_incompatible, send_pipes, recv_pipes, zfs_compressed, force, guid_check, + use_bookmarks, bookmark_tag, property_format, no_clone=True, + target_path=None, strip_path=0, required_snapshots=None): + """sync source_dataset's snapshots to target_dataset, while also thinning out old snapshots along the way. + + :type source_dataset: ZfsContainer + :type target_dataset: ZfsContainer + :type features: list[str] + :type show_progress: bool + :type filter_properties: list[str] + :type set_properties: list[str] + :type ignore_recv_exit_code: bool + :type holds: bool + :type rollback: bool + :type decrypt: bool + :type encrypt: bool + :type also_other_snapshots: bool + :type no_send: bool + :type destroy_incompatible: bool + :type send_pipes: list[str] + :type recv_pipes: list[str] + :type zfs_compressed: bool + :type force: bool + :type guid_check: bool + :type use_bookmarks: bool + :type bookmark_tag: str + :type property_format: str + :type no_clone: bool + :type target_path: str + :type strip_path: int + :type required_snapshots: set[ZfsSnapshot]|None + """ + + # determine how to handle encryption during the transfer: + send_properties = True + raw = False + + # source dataset encrypted? + if source_dataset.properties.get('encryption', 'off') != 'off': + # user wants to send it over decrypted? + if decrypt: + # when decrypting, zfs can't send properties + send_properties = False + else: + # keep data encrypted by sending it raw (including properties) + raw = True + + (source_common_snapshot, is_clone_origin, source_obsoletes, target_obsoletes, target_transfers, + incompatible_target_snapshots) = \ + _plan_sync(source_dataset, target_dataset=target_dataset, also_other_snapshots=also_other_snapshots, + guid_check=guid_check, raw=raw, bookmark_tag=bookmark_tag, + no_clone=no_clone, target_path=target_path, strip_path=strip_path, + required_snapshots=required_snapshots) + + source_dataset.zfs_node.logger.progress("Pre-cleaning..") + + # NOTE: we do a pre-clean because we dont want filesystems to fillup when backups keep failing. + # Also usefull with no_send to still cleanup stuff. + _pre_clean( + source_dataset, target_dataset=target_dataset, source_common_snapshot=source_common_snapshot, + source_obsoletes=source_obsoletes, target_obsoletes=target_obsoletes, target_transfers=target_transfers) + + # check if we can resume + if len(target_transfers) > 0: + source_dataset.zfs_node.logger.progress("Verifying resume token...") + + resume_token = _validate_resume_token(source_dataset, target_dataset, target_transfers[0]) + else: + resume_token = None + + source_dataset.zfs_node.logger.progress("Preparing...") + + # handle incompatible stuff on target + _handle_incompatible_target(target_dataset, incompatible_target_snapshots, destroy_incompatible, force, + source_common_snapshot, is_clone_origin, resume_token is not None) + + # now actually transfer the snapshots, if we want + if no_send or len(target_transfers) == 0: + return + + (active_filter_properties, active_set_properties) = _active_properties(source_dataset, filter_properties, + set_properties, property_format) + + # encrypt at target? + write_embedded = True + if encrypt and not raw: + # filter out encryption properties to let encryption on the target take place + active_filter_properties.extend(["keylocation", "pbkdf2iters", "keyformat", "encryption"]) + write_embedded = False + + # the source-side point-in-time (snapshot, bookmark or clone origin) to use as base for the + # next incremental send + incremental_base = source_common_snapshot + + # the target-side counterpart of the common snapshot. (a clone origin has no counterpart here: + # it lives on the origin's target dataset) + if is_clone_origin: + prev_target_snapshot = None + else: + prev_target_snapshot = target_dataset.find_snapshot(source_common_snapshot) + + do_rollback = rollback + for target_snapshot in target_transfers: + + source_snapshot = source_dataset.find_snapshot(target_snapshot) + assert source_snapshot is not None + + # do the rollback, one time at first transfer + if do_rollback: + source_dataset.zfs_node.logger.progress("Rolling back {}...".format(target_dataset)) + + target_dataset.rollback() + do_rollback = False + + source_snapshot.transfer_snapshot(target_snapshot, features=features, + prev_snapshot=incremental_base, show_progress=show_progress, + filter_properties=active_filter_properties, + set_properties=active_set_properties, + ignore_recv_exit_code=ignore_recv_exit_code, + resume_token=resume_token, write_embedded=write_embedded, raw=raw, + send_properties=send_properties, send_pipes=send_pipes, + recv_pipes=recv_pipes, zfs_compressed=zfs_compressed, force=force) + + # the resume token is only valid for the first transfer + resume_token = None + + # hold/release common snapshot on the target. + if holds: + target_snapshot.hold() + + if prev_target_snapshot: + prev_target_snapshot.release() + + # cleanup on the source and determine the base for the next incremental send + incremental_base = _post_transfer_source_cleanup(source_snapshot, incremental_base, source_obsoletes, + holds, use_bookmarks, bookmark_tag) + + # destroy the previous target snapshot if obsolete (usually this is only the common_snapshot, + # the rest was already destroyed or will not be send) + if prev_target_snapshot is not None and prev_target_snapshot in target_obsoletes: + prev_target_snapshot.destroy() + + prev_target_snapshot = target_snapshot diff --git a/zfs_autobackup/util.py b/zfs_autobackup/util.py index db6f07c..c5e6532 100644 --- a/zfs_autobackup/util.py +++ b/zfs_autobackup/util.py @@ -1,4 +1,3 @@ - # NOTE: surprisingly sha1 in via python3 is faster than the native sha1sum utility, even in the way we use below! import os import platform @@ -9,19 +8,21 @@ from datetime import datetime def tmp_name(suffix=""): """create temporary name unique to this process and node. always retruns the same result during the same execution""" - #we could use uuids but those are ugly and confusing - name="{}-{}-{}".format( - os.path.basename(sys.argv[0]).replace(" ","_"), + # we could use uuids but those are ugly and confusing + name = "{}-{}-{}".format( + os.path.basename(sys.argv[0]).replace(" ", "_"), platform.node(), os.getpid()) - name=name+suffix + name = name + suffix return name def get_tmp_clone_name(snapshot): - pool=snapshot.zfs_node.get_pool(snapshot) - return pool.name+"/"+tmp_name() - + """ + :type snapshot: ZfsSnapshot + """ + pool = snapshot.zfs_node.get_pool(snapshot) + return pool.name + "/" + tmp_name() def output_redir(): @@ -33,10 +34,12 @@ def output_redir(): os.dup2(devnull, sys.stdout.fileno()) os.dup2(devnull, sys.stderr.fileno()) + def sigpipe_handler(sig, stack): - #redir output so we dont get more SIGPIPES during cleanup. (which my try to write to stdout) + # redir output so we dont get more SIGPIPES during cleanup. (which my try to write to stdout) output_redir() - #deb('redir') + # deb('redir') + # def check_output(): # """make sure stdout still functions. if its broken, this will trigger a SIGPIPE which will be handled by the sigpipe_handler.""" @@ -55,9 +58,11 @@ def sigpipe_handler(sig, stack): # This function will be mocked during unit testing. -datetime_now_mock=None +datetime_now_mock = None + + def datetime_now(utc): if datetime_now_mock is None: - return( datetime.utcnow() if utc else datetime.now()) + return (datetime.utcnow() if utc else datetime.now()) else: return datetime_now_mock