From 8b8be80ab72ea9d56fbb1d819e0c66e90564d858 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Mon, 2 Oct 2023 23:15:37 +0200 Subject: [PATCH] tests can be run in a dockercontainer now. (just start ./tests/run_tests_docker to magically do it) changed time patching during testing to use mocktime() instead. fixed alpine issues. fixed #206 --- tests/Dockerfile | 17 +++++ tests/autorun_tests_docker | 3 + tests/basetest.py | 13 ++-- tests/run_tests | 2 + tests/run_tests_docker | 16 +++++ tests/test_cmdpipe.py | 26 ++++---- tests/test_destroymissing.py | 4 +- tests/test_encryption.py | 20 +++--- tests/test_executenode.py | 14 ++-- tests/test_externalfailures.py | 78 +++++++--------------- tests/test_regressions.py | 8 +-- tests/test_scaling.py | 8 +-- tests/test_sendrecvpipes.py | 20 +++--- tests/test_thinner.py | 4 +- tests/test_verify.py | 2 +- tests/test_zfsautobackup.py | 113 ++++++++++++++++---------------- tests/test_zfsautobackup31.py | 16 ++--- tests/test_zfsautobackup32.py | 18 ++--- tests/test_zfscheck.py | 17 +++-- tests/tests_docker | 27 ++++++++ zfs_autobackup/ZfsAutoverify.py | 4 +- zfs_autobackup/ZfsCheck.py | 2 +- zfs_autobackup/ZfsDataset.py | 4 +- zfs_autobackup/ZfsNode.py | 2 +- 24 files changed, 236 insertions(+), 202 deletions(-) create mode 100644 tests/Dockerfile create mode 100755 tests/autorun_tests_docker create mode 100755 tests/run_tests_docker create mode 100755 tests/tests_docker diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 0000000..55aeed0 --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,17 @@ +FROM alpine:3.18 + + +#base packages +RUN apk update +RUN apk add py3-pip + +#zfs autobackup tests dependencies +RUN apk add zfs openssh lzop pigz zstd gzip xz lz4 mbuffer udev zfs-udev + + +#python modules +COPY requirements.txt / +RUN pip3 install -r requirements.txt + +#git repo should be mounted in /app: +ENTRYPOINT [ "/app/tests/tests_docker" ] diff --git a/tests/autorun_tests_docker b/tests/autorun_tests_docker new file mode 100755 index 0000000..75c921f --- /dev/null +++ b/tests/autorun_tests_docker @@ -0,0 +1,3 @@ +#!/bin/sh + +find tests zfs_autobackup -name '*.py' |entr ./tests/run_tests_docker $@ diff --git a/tests/basetest.py b/tests/basetest.py index 18c15fd..d507d8b 100644 --- a/tests/basetest.py +++ b/tests/basetest.py @@ -34,14 +34,13 @@ import datetime TEST_POOLS="test_source1 test_source2 test_target1" -ZFS_USERSPACE= subprocess.check_output("dpkg-query -W zfsutils-linux |cut -f2", shell=True).decode('utf-8').rstrip() -ZFS_KERNEL= subprocess.check_output("modinfo zfs|grep ^version |sed 's/.* //'", shell=True).decode('utf-8').rstrip() +# ZFS_USERSPACE= subprocess.check_output("dpkg-query -W zfsutils-linux |cut -f2", shell=True).decode('utf-8').rstrip() +# ZFS_KERNEL= subprocess.check_output("modinfo zfs|grep ^version |sed 's/.* //'", shell=True).decode('utf-8').rstrip() print("###########################################") print("#### Unit testing against:") -print("#### Python :"+sys.version.replace("\n", " ")) -print("#### ZFS userspace :"+ZFS_USERSPACE) -print("#### ZFS kernel :"+ZFS_KERNEL) +print("#### Python : "+sys.version.replace("\n", " ")) +print("#### ZFS version : "+subprocess.check_output("zfs --version", shell=True).decode('utf-8').rstrip().replace('\n', ' ')) print("#############################################") @@ -78,7 +77,7 @@ def redirect_stderr(target): def shelltest(cmd): """execute and print result as nice copypastable string for unit tests (adds extra newlines on top/bottom)""" - ret=(subprocess.check_output("SUDO_ASKPASS=./password.sh sudo -A "+cmd , shell=True).decode('utf-8')) + ret=(subprocess.check_output(cmd , shell=True).decode('utf-8')) print("######### result of: {}".format(cmd)) print(ret) @@ -90,7 +89,7 @@ def prepare_zpools(): print("Preparing zfs filesystems...") #need ram blockdevice - subprocess.check_call("modprobe brd rd_size=512000", shell=True) + # subprocess.check_call("modprobe brd rd_size=512000", shell=True) #remove old stuff subprocess.call("zpool destroy test_source1 2>/dev/null", shell=True) diff --git a/tests/run_tests b/tests/run_tests index b2347d4..78c1c6a 100755 --- a/tests/run_tests +++ b/tests/run_tests @@ -18,6 +18,8 @@ if ! [ -e /root/.ssh/id_rsa ]; then ssh -oStrictHostKeyChecking=no localhost true || exit 1 fi +modprobe brd rd_size=512000 + umount /tmp/ZfsCheck* coverage run --branch --source zfs_autobackup -m unittest discover -vvvvf $SCRIPTDIR $@ 2>&1 diff --git a/tests/run_tests_docker b/tests/run_tests_docker new file mode 100755 index 0000000..9d33960 --- /dev/null +++ b/tests/run_tests_docker @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +#remove stuff from previous local tests +zpool destroy test_source1 2>/dev/null || true +zpool destroy test_source2 2>/dev/null || true +zpool destroy test_target1 2>/dev/null || true + +#is needed +modprobe brd rd_size=512000 || true + +# builds and starts a docker container to run the test suite +docker build -t zfs-autobackup-test -f tests/Dockerfile . +docker run --name zfs-autobackup-test --privileged --rm -it -v .:/app zfs-autobackup-test $@ + diff --git a/tests/test_cmdpipe.py b/tests/test_cmdpipe.py index a65ed57..d127235 100644 --- a/tests/test_cmdpipe.py +++ b/tests/test_cmdpipe.py @@ -9,11 +9,11 @@ class TestCmdPipe(unittest2.TestCase): p=CmdPipe(readonly=False, inp=None) err=[] out=[] - p.add(CmdItem(["ls", "-d", "/", "/", "/nonexistent"], stderr_handler=lambda line: err.append(line), exit_handler=lambda exit_code: self.assertEqual(exit_code,2), stdout_handler=lambda line: out.append(line))) + p.add(CmdItem(["sh", "-c", "echo out1;echo err1 >&2; echo out2; echo err2 >&2"], stderr_handler=lambda line: err.append(line), exit_handler=lambda exit_code: self.assertEqual(exit_code,0), stdout_handler=lambda line: out.append(line))) executed=p.execute() - self.assertEqual(err, ["ls: cannot access '/nonexistent': No such file or directory"]) - self.assertEqual(out, ["/","/"]) + self.assertEqual(out, ["out1", "out2"]) + self.assertEqual(err, ["err1","err2"]) self.assertIsNone(executed) def test_input(self): @@ -56,16 +56,16 @@ class TestCmdPipe(unittest2.TestCase): err2=[] err3=[] out=[] - p.add(CmdItem(["ls", "/nonexistent1"], stderr_handler=lambda line: err1.append(line), exit_handler=lambda exit_code: self.assertEqual(exit_code,2))) - p.add(CmdItem(["ls", "/nonexistent2"], stderr_handler=lambda line: err2.append(line), exit_handler=lambda exit_code: self.assertEqual(exit_code,2))) - p.add(CmdItem(["ls", "/nonexistent3"], stderr_handler=lambda line: err3.append(line), exit_handler=lambda exit_code: self.assertEqual(exit_code,2), stdout_handler=lambda line: out.append(line))) + p.add(CmdItem(["sh", "-c", "echo err1 >&2"], stderr_handler=lambda line: err1.append(line), )) + p.add(CmdItem(["sh", "-c", "echo err2 >&2"], stderr_handler=lambda line: err2.append(line), )) + p.add(CmdItem(["sh", "-c", "echo err3 >&2"], stderr_handler=lambda line: err3.append(line), stdout_handler=lambda line: out.append(line))) executed=p.execute() - self.assertEqual(err1, ["ls: cannot access '/nonexistent1': No such file or directory"]) - self.assertEqual(err2, ["ls: cannot access '/nonexistent2': No such file or directory"]) - self.assertEqual(err3, ["ls: cannot access '/nonexistent3': No such file or directory"]) + self.assertEqual(err1, ["err1"]) + self.assertEqual(err2, ["err2"]) + self.assertEqual(err3, ["err3"]) self.assertEqual(out, []) - self.assertIsNone(executed) + self.assertTrue(executed) def test_exitcode(self): """test piped exitcodes """ @@ -74,9 +74,9 @@ class TestCmdPipe(unittest2.TestCase): err2=[] err3=[] out=[] - p.add(CmdItem(["bash", "-c", "exit 1"], stderr_handler=lambda line: err1.append(line), exit_handler=lambda exit_code: self.assertEqual(exit_code,1))) - p.add(CmdItem(["bash", "-c", "exit 2"], stderr_handler=lambda line: err2.append(line), exit_handler=lambda exit_code: self.assertEqual(exit_code,2))) - p.add(CmdItem(["bash", "-c", "exit 3"], stderr_handler=lambda line: err3.append(line), exit_handler=lambda exit_code: self.assertEqual(exit_code,3), stdout_handler=lambda line: out.append(line))) + p.add(CmdItem(["sh", "-c", "exit 1"], stderr_handler=lambda line: err1.append(line), exit_handler=lambda exit_code: self.assertEqual(exit_code,1))) + p.add(CmdItem(["sh", "-c", "exit 2"], stderr_handler=lambda line: err2.append(line), exit_handler=lambda exit_code: self.assertEqual(exit_code,2))) + p.add(CmdItem(["sh", "-c", "exit 3"], stderr_handler=lambda line: err3.append(line), exit_handler=lambda exit_code: self.assertEqual(exit_code,3), stdout_handler=lambda line: out.append(line))) executed=p.execute() self.assertEqual(err1, []) diff --git a/tests/test_destroymissing.py b/tests/test_destroymissing.py index c74f9c7..89eb4ec 100644 --- a/tests/test_destroymissing.py +++ b/tests/test_destroymissing.py @@ -13,10 +13,10 @@ class TestZfsNode(unittest2.TestCase): def test_destroymissing(self): #initial backup - with patch('time.strftime', return_value="test-19101111000000"): #1000 years in past + with mocktime("19101111000000"): #1000 years in past self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds".split(" ")).run()) - with patch('time.strftime', return_value="test-20101111000000"): #far in past + with mocktime("20101111000000"): #far in past self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) diff --git a/tests/test_encryption.py b/tests/test_encryption.py index 89f8c68..11114e3 100644 --- a/tests/test_encryption.py +++ b/tests/test_encryption.py @@ -49,11 +49,11 @@ class TestZfsEncryption(unittest2.TestCase): self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsourcekeyless", unload_key=True) # raw mode shouldn't need a key self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split(" ")).run()) - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split(" ")).run()) @@ -86,11 +86,11 @@ test_target1/test_source2/fs2/sub encryption self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received".split(" ")).run()) - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received".split(" ")).run()) @@ -121,11 +121,11 @@ test_target1/test_source2/fs2/sub encryptionroot - self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split(" ")).run()) - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split(" ")).run()) @@ -156,14 +156,14 @@ test_target1/test_source2/fs2/sub encryptionroot - self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) self.assertFalse(ZfsAutobackup( "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split( " ")).run()) - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received".split(" ")).run()) self.assertFalse(ZfsAutobackup( @@ -199,13 +199,13 @@ test_target1/test_source2/fs2/sub encryptionroot - self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress".split(" ")).run()) #this is invalid in raw mode shelltest("zfs snapshot test_target1/test_source1/fs1/encryptedsource@incompatible") - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): #should fail because of incompatble snapshot self.assertEqual(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty".split(" ")).run(),1) #should destroy incompatible and continue diff --git a/tests/test_executenode.py b/tests/test_executenode.py index f05a6da..1b1db1f 100644 --- a/tests/test_executenode.py +++ b/tests/test_executenode.py @@ -33,9 +33,9 @@ class TestExecuteNode(unittest2.TestCase): #return std err as well, trigger stderr by listing something non existing with self.subTest("stderr return"): - (stdout, stderr)=node.run(["ls", "nonexistingfile"], return_stderr=True, valid_exitcodes=[2]) + (stdout, stderr)=node.run(["sh", "-c", "echo bla >&2"], return_stderr=True, valid_exitcodes=[0]) self.assertEqual(stdout,[]) - self.assertRegex(stderr[0],"nonexistingfile") + self.assertRegex(stderr[0],"bla") #slow command, make sure things dont exit too early with self.subTest("early exit test"): @@ -110,19 +110,17 @@ class TestExecuteNode(unittest2.TestCase): with self.subTest("check stderr on pipe output side"): output=nodea.run(["true"], pipe=True, valid_exitcodes=[0]) - (stdout, stderr)=nodeb.run(["ls", "nonexistingfile"], inp=output, return_stderr=True, valid_exitcodes=[2]) + (stdout, stderr)=nodeb.run(["sh", "-c", "echo bla >&2"], inp=output, return_stderr=True, valid_exitcodes=[0]) self.assertEqual(stdout,[]) - self.assertRegex(stderr[0], "nonexistingfile" ) + self.assertRegex(stderr[0], "bla" ) with self.subTest("check stderr on pipe input side (should be only printed)"): - output=nodea.run(["ls", "nonexistingfile"], pipe=True, valid_exitcodes=[2]) + output=nodea.run(["sh", "-c", "echo bla >&2"], pipe=True, valid_exitcodes=[0]) (stdout, stderr)=nodeb.run(["true"], inp=output, return_stderr=True, valid_exitcodes=[0]) self.assertEqual(stdout,[]) self.assertEqual(stderr,[]) - - def test_pipe_local_local(self): nodea=ExecuteNode(debug_output=True) nodeb=ExecuteNode(debug_output=True) @@ -209,5 +207,3 @@ class TestExecuteNode(unittest2.TestCase): -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_externalfailures.py b/tests/test_externalfailures.py index 821c718..7c1dcd1 100644 --- a/tests/test_externalfailures.py +++ b/tests/test_externalfailures.py @@ -32,7 +32,7 @@ class TestExternalFailures(unittest2.TestCase): def test_initial_resume(self): # inital backup, leaves resume token - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.generate_resume() # --test should resume and succeed @@ -42,12 +42,7 @@ class TestExternalFailures(unittest2.TestCase): print(buf.getvalue()) - # did we really resume? - if "0.6.5" in ZFS_USERSPACE: - # abort this late, for beter coverage - self.skipTest("Resume not supported in this ZFS userspace version") - else: - self.assertIn(": resuming", buf.getvalue()) + self.assertIn(": resuming", buf.getvalue()) # should resume and succeed with OutputIO() as buf: @@ -56,12 +51,7 @@ class TestExternalFailures(unittest2.TestCase): print(buf.getvalue()) - # did we really resume? - if "0.6.5" in ZFS_USERSPACE: - # abort this late, for beter coverage - self.skipTest("Resume not supported in this ZFS userspace version") - else: - self.assertIn(": resuming", buf.getvalue()) + self.assertIn(": resuming", buf.getvalue()) r = shelltest("zfs list -H -o name -r -t all test_target1") self.assertMultiLineEqual(r, """ @@ -81,11 +71,11 @@ test_target1/test_source2/fs2/sub@test-20101111000000 def test_incremental_resume(self): # initial backup - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) # incremental backup leaves resume token - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.generate_resume() # --test should resume and succeed @@ -95,12 +85,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 print(buf.getvalue()) - # did we really resume? - if "0.6.5" in ZFS_USERSPACE: - # abort this late, for beter coverage - self.skipTest("Resume not supported in this ZFS userspace version") - else: - self.assertIn(": resuming", buf.getvalue()) + self.assertIn(": resuming", buf.getvalue()) # should resume and succeed with OutputIO() as buf: @@ -110,11 +95,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 print(buf.getvalue()) # did we really resume? - if "0.6.5" in ZFS_USERSPACE: - # abort this late, for beter coverage - self.skipTest("Resume not supported in this ZFS userspace version") - else: - self.assertIn(": resuming", buf.getvalue()) + self.assertIn(": resuming", buf.getvalue()) r = shelltest("zfs list -H -o name -r -t all test_target1") self.assertMultiLineEqual(r, """ @@ -134,11 +115,9 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # generate an invalid resume token, and verify if its aborted automaticly def test_initial_resumeabort(self): - if "0.6.5" in ZFS_USERSPACE: - self.skipTest("Resume not supported in this ZFS userspace version") # inital backup, leaves resume token - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.generate_resume() # remove corresponding source snapshot, so it becomes invalid @@ -148,11 +127,11 @@ test_target1/test_source2/fs2/sub@test-20101111000000 shelltest("zfs destroy test_target1/test_source1/fs1/sub; true") # --test try again, should abort old resume - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test".split(" ")).run()) # try again, should abort old resume - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) r = shelltest("zfs list -H -o name -r -t all test_target1") @@ -172,26 +151,23 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # generate an invalid resume token, and verify if its aborted automaticly def test_incremental_resumeabort(self): - if "0.6.5" in ZFS_USERSPACE: - self.skipTest("Resume not supported in this ZFS userspace version") - # initial backup - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) # icremental backup, leaves resume token - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.generate_resume() # remove corresponding source snapshot, so it becomes invalid shelltest("zfs destroy test_source1/fs1@test-20101111000001") # --test try again, should abort old resume - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test".split(" ")).run()) # try again, should abort old resume - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) r = shelltest("zfs list -H -o name -r -t all test_target1") @@ -212,22 +188,19 @@ test_target1/test_source2/fs2/sub@test-20101111000000 # create a resume situation, where the other side doesnt want the snapshot anymore ( should abort resume ) def test_abort_unwanted_resume(self): - if "0.6.5" in ZFS_USERSPACE: - self.skipTest("Resume not supported in this ZFS userspace version") - - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) # generate resume - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.generate_resume() with OutputIO() as buf: with redirect_stdout(buf): # incremental, doesnt want previous anymore - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( - "test test_target1 --no-progress --verbose --keep-target=0 --allow-empty".split(" ")).run()) + "test test_target1 --no-progress --verbose --keep-target=0 --allow-empty --debug".split(" ")).run()) print(buf.getvalue()) @@ -250,14 +223,11 @@ test_target1/test_source2/fs2/sub@test-20101111000002 # test with empty snapshot list (this was a bug) def test_abort_resume_emptysnapshotlist(self): - if "0.6.5" in ZFS_USERSPACE: - self.skipTest("Resume not supported in this ZFS userspace version") - - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) # generate resume - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.generate_resume() shelltest("zfs destroy test_source1/fs1@test-20101111000001") @@ -265,7 +235,7 @@ test_target1/test_source2/fs2/sub@test-20101111000002 with OutputIO() as buf: with redirect_stdout(buf): # incremental, doesnt want previous anymore - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( "test test_target1 --no-progress --verbose --no-snapshot".split( " ")).run()) @@ -277,14 +247,14 @@ test_target1/test_source2/fs2/sub@test-20101111000002 def test_missing_common(self): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) # remove common snapshot and leave nothing shelltest("zfs release zfs_autobackup:test test_source1/fs1@test-20101111000000") shelltest("zfs destroy test_source1/fs1@test-20101111000000") - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertTrue(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) #UPDATE: offcourse the one thing that wasn't tested had a bug :( (in ExecuteNode.run()). @@ -295,7 +265,7 @@ test_target1/test_source2/fs2/sub@test-20101111000002 # #recreate target pool without any features # # shelltest("zfs set compress=on test_source1; zpool destroy test_target1; zpool create test_target1 -o feature@project_quota=disabled /dev/ram2") # -# with patch('time.strftime', return_value="test-20101111000000"): +# with mocktime("20101111000000"): # self.assertFalse(ZfsAutobackup("test test_target1 --verbose --allow-empty --no-progress".split(" ")).run()) # # r = shelltest("zfs list -H -o name -r -t all test_target1") diff --git a/tests/test_regressions.py b/tests/test_regressions.py index 4592599..7a5dac9 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -11,17 +11,17 @@ class TestZfsNode(unittest2.TestCase): def test_keepsource0target10queuedsend(self): """Test if thinner doesnt destroy too much early on if there are no common snapshots YET. Issue #84""" - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send".split( " ")).run()) - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty --no-send".split( " ")).run()) - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( "test test_target1 --no-progress --verbose --keep-source=0 --keep-target=10 --allow-empty".split( " ")).run()) @@ -65,7 +65,7 @@ test_target1/test_source2/fs2/sub@test-20101111000002 shelltest("zfs set autobackup:test=true test_target1/target_shouldnotbeexcluded") shelltest("zfs create test_target1/target") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( "test test_target1/target --no-progress --verbose --allow-empty".split( " ")).run()) diff --git a/tests/test_scaling.py b/tests/test_scaling.py index 854e00e..b93d9a8 100644 --- a/tests/test_scaling.py +++ b/tests/test_scaling.py @@ -33,7 +33,7 @@ class TestZfsScaling(unittest2.TestCase): run_counter=0 with patch.object(ExecuteNode,'run', run_count) as p: - with patch('time.strftime', return_value="test-20101112000000"): + with mocktime("20101112000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --keep-source=10000 --keep-target=10000 --no-holds --allow-empty".split(" ")).run()) @@ -47,7 +47,7 @@ class TestZfsScaling(unittest2.TestCase): run_counter=0 with patch.object(ExecuteNode,'run', run_count) as p: - with patch('time.strftime', return_value="test-20101112000001"): + with mocktime("20101112000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --keep-source=10000 --keep-target=10000 --no-holds --allow-empty".split(" ")).run()) @@ -75,7 +75,7 @@ class TestZfsScaling(unittest2.TestCase): run_counter=0 with patch.object(ExecuteNode,'run', run_count) as p: - with patch('time.strftime', return_value="test-20101112000000"): + with mocktime("20101112000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) @@ -90,7 +90,7 @@ class TestZfsScaling(unittest2.TestCase): run_counter=0 with patch.object(ExecuteNode,'run', run_count) as p: - with patch('time.strftime', return_value="test-20101112000001"): + with mocktime("20101112000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --no-holds --allow-empty".split(" ")).run()) diff --git a/tests/test_sendrecvpipes.py b/tests/test_sendrecvpipes.py index 511b5c3..c552540 100644 --- a/tests/test_sendrecvpipes.py +++ b/tests/test_sendrecvpipes.py @@ -14,7 +14,7 @@ class TestSendRecvPipes(unittest2.TestCase): """send basics (remote/local send pipe)""" with self.subTest("local local pipe"): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", "--clear-mountpoint", "--send-pipe=dd bs=1M", "--recv-pipe=dd bs=2M"]).run()) @@ -22,7 +22,7 @@ class TestSendRecvPipes(unittest2.TestCase): shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") with self.subTest("remote local pipe"): - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", "--ssh-source=localhost", "--send-pipe=dd bs=1M", "--recv-pipe=dd bs=2M"]).run()) @@ -30,7 +30,7 @@ class TestSendRecvPipes(unittest2.TestCase): shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") with self.subTest("local remote pipe"): - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", "--ssh-target=localhost", "--send-pipe=dd bs=1M", "--recv-pipe=dd bs=2M"]).run()) @@ -38,7 +38,7 @@ class TestSendRecvPipes(unittest2.TestCase): shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") with self.subTest("remote remote pipe"): - with patch('time.strftime', return_value="test-20101111000003"): + with mocktime("20101111000003"): self.assertFalse(ZfsAutobackup( ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", "--ssh-source=localhost", "--ssh-target=localhost", "--send-pipe=dd bs=1M", @@ -72,7 +72,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 for compress in zfs_autobackup.compressors.COMPRESS_CMDS.keys(): with self.subTest("compress " + compress): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( ["test", "test_target1", "--exclude-received", "--no-holds", "--no-progress", "--verbose", "--compress=" + compress]).run()) @@ -83,14 +83,14 @@ test_target1/test_source2/fs2/sub@test-20101111000003 """test different buffer configurations""" with self.subTest("local local pipe"): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", "--clear-mountpoint", "--buffer=1M"]).run()) shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") with self.subTest("remote local pipe"): - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( ["test", "test_target1", "--allow-empty", "--verbose", "--exclude-received", "--no-holds", "--no-progress", "--ssh-source=localhost", "--buffer=1M"]).run()) @@ -98,7 +98,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") with self.subTest("local remote pipe"): - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup( ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", "--ssh-target=localhost", "--buffer=1M"]).run()) @@ -106,7 +106,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 shelltest("zfs destroy -r test_target1/test_source1/fs1/sub") with self.subTest("remote remote pipe"): - with patch('time.strftime', return_value="test-20101111000003"): + with mocktime("20101111000003"): self.assertFalse(ZfsAutobackup( ["test", "test_target1", "--allow-empty", "--exclude-received", "--no-holds", "--no-progress", "--ssh-source=localhost", "--ssh-target=localhost", "--buffer=1M"]).run()) @@ -138,7 +138,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 """test rate limit""" start = time.time() - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( ["test", "test_target1", "--exclude-received", "--no-holds", "--no-progress", "--rate=50k"]).run()) diff --git a/tests/test_thinner.py b/tests/test_thinner.py index 3de2120..4425d3d 100644 --- a/tests/test_thinner.py +++ b/tests/test_thinner.py @@ -85,7 +85,7 @@ class TestThinner(unittest2.TestCase): if random.random()>=0.5: things.append(Thing(now)) - (keeps, removes)=thinner.thin(things, now=now) + (keeps, removes)=thinner.thin(things, keep_objects=[], now=now) things=keeps @@ -143,7 +143,7 @@ class TestThinner(unittest2.TestCase): if random.random()>=0.5: things.append(Thing(now)) - (things, removes)=thinner.thin(things, now=now) + (things, removes)=thinner.thin(things, keep_objects=[], now=now) result=[] for thing in things: diff --git a/tests/test_verify.py b/tests/test_verify.py index a1c80f2..5ecd402 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -38,7 +38,7 @@ class TestZfsVerify(unittest2.TestCase): shelltest("dd if=/dev/urandom of=/dev/zvol/test_source1/fs1/bad_zvol count=1 bs=512k") #create backup - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --no-holds".split(" ")).run()) #Do an ugly hack to create a fault in the bad filesystem diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index d3b418f..fac8db2 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -35,7 +35,7 @@ class TestZfsAutobackup(unittest2.TestCase): def test_snapshotmode(self): """test snapshot tool mode""" - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test --no-progress --verbose".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -55,11 +55,12 @@ test_target1 """) def test_defaults(self): + self.maxDiff=2000 with self.subTest("no datasets selected"): with OutputIO() as buf: with redirect_stderr(buf): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertTrue(ZfsAutobackup("nonexisting test_target1 --verbose --debug --no-progress".split(" ")).run()) print(buf.getvalue()) @@ -69,7 +70,7 @@ test_target1 with self.subTest("defaults with full verbose and debug"): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --debug --no-progress".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -98,7 +99,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 """) with self.subTest("bare defaults, allow empty"): - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --no-progress".split(" ")).run()) @@ -168,47 +169,43 @@ test_target1/test_source2/fs2/sub@test-20101111000001 userrefs 1 - """) #make sure time handling is correctly. try to make snapshots a year appart and verify that only snapshots mostly 1y old are kept + #So in this case we only want to see 2 snapshots of 2011, and none of the 2010's anymore. with self.subTest("test time checking"): - with patch('time.strftime', return_value="test-20111111000000"): + with mocktime("20111211000000"): self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --verbose --no-progress".split(" ")).run()) - - time_str="20111112000000" #month in the "future" - future_timestamp=time_secs=time.mktime(time.strptime(time_str,"%Y%m%d%H%M%S")) - with patch('time.time', return_value=future_timestamp): - with patch('time.strftime', return_value="test-20111111000001"): + with mocktime("20111211000001"): self.assertFalse(ZfsAutobackup("test test_target1 --allow-empty --verbose --keep-source 1y1y --keep-target 1d1y --no-progress".split(" ")).run()) - r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) self.assertMultiLineEqual(r,""" test_source1 test_source1/fs1 -test_source1/fs1@test-20111111000000 -test_source1/fs1@test-20111111000001 +test_source1/fs1@test-20111211000000 +test_source1/fs1@test-20111211000001 test_source1/fs1/sub -test_source1/fs1/sub@test-20111111000000 -test_source1/fs1/sub@test-20111111000001 +test_source1/fs1/sub@test-20111211000000 +test_source1/fs1/sub@test-20111211000001 test_source2 test_source2/fs2 test_source2/fs2/sub -test_source2/fs2/sub@test-20111111000000 -test_source2/fs2/sub@test-20111111000001 +test_source2/fs2/sub@test-20111211000000 +test_source2/fs2/sub@test-20111211000001 test_source2/fs3 test_source2/fs3/sub test_target1 test_target1/test_source1 test_target1/test_source1/fs1 -test_target1/test_source1/fs1@test-20111111000000 -test_target1/test_source1/fs1@test-20111111000001 +test_target1/test_source1/fs1@test-20111211000000 +test_target1/test_source1/fs1@test-20111211000001 test_target1/test_source1/fs1/sub -test_target1/test_source1/fs1/sub@test-20111111000000 -test_target1/test_source1/fs1/sub@test-20111111000001 +test_target1/test_source1/fs1/sub@test-20111211000000 +test_target1/test_source1/fs1/sub@test-20111211000001 test_target1/test_source2 test_target1/test_source2/fs2 test_target1/test_source2/fs2/sub -test_target1/test_source2/fs2/sub@test-20111111000000 -test_target1/test_source2/fs2/sub@test-20111111000001 +test_target1/test_source2/fs2/sub@test-20111211000000 +test_target1/test_source2/fs2/sub@test-20111211000001 """) def test_ignore_othersnaphots(self): @@ -216,7 +213,7 @@ test_target1/test_source2/fs2/sub@test-20111111000001 r=shelltest("zfs snapshot test_source1/fs1@othersimple") r=shelltest("zfs snapshot test_source1/fs1@otherdate-20001111000000") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -251,7 +248,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 r=shelltest("zfs snapshot test_source1/fs1@othersimple") r=shelltest("zfs snapshot test_source1/fs1@otherdate-20001111000000") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --other-snapshots".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -286,7 +283,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 def test_nosnapshot(self): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --no-progress".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -310,7 +307,7 @@ test_target1/test_source2/fs2 def test_nosend(self): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-send --no-progress".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -333,7 +330,7 @@ test_target1 def test_ignorereplicated(self): r=shelltest("zfs snapshot test_source1/fs1@otherreplication") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --ignore-replicated".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -362,7 +359,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 def test_noholds(self): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-holds --no-progress".split(" ")).run()) r=shelltest("zfs get -r userrefs test_source1 test_source2 test_target1") @@ -394,7 +391,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 userrefs 0 - def test_strippath(self): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --strip-path=1 --no-progress".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -437,10 +434,10 @@ test_target1/fs2/sub@test-20101111000000 r=shelltest("zfs set refreservation=1M test_source1/fs1") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --clear-refreservation".split(" ")).run()) - r=shelltest("zfs get refreservation -r test_source1 test_source2 test_target1") + r=shelltest("zfs get -r refreservation test_source1 test_source2 test_target1") self.assertMultiLineEqual(r,""" NAME PROPERTY VALUE SOURCE test_source1 refreservation none default @@ -475,10 +472,10 @@ test_target1/test_source2/fs2/sub@test-20101111000000 refreservation - self.skipTest("This zfs-userspace version doesnt support -o") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --clear-mountpoint --debug".split(" ")).run()) - r=shelltest("zfs get canmount -r test_source1 test_source2 test_target1") + r=shelltest("zfs get -r canmount test_source1 test_source2 test_target1") self.assertMultiLineEqual(r,""" NAME PROPERTY VALUE SOURCE test_source1 canmount on default @@ -508,17 +505,17 @@ test_target1/test_source2/fs2/sub@test-20101111000000 canmount - - def test_rollback(self): #initial backup - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) #make change r=shelltest("touch /test_target1/test_source1/fs1/change.txt") - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): #should fail (busy) self.assertTrue(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): #rollback, should succeed self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --rollback".split(" ")).run()) @@ -526,14 +523,14 @@ test_target1/test_source2/fs2/sub@test-20101111000000 canmount - - def test_destroyincompat(self): #initial backup - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) #add multiple compatible snapshot (written is still 0) r=shelltest("zfs snapshot test_target1/test_source1/fs1@compatible1") r=shelltest("zfs snapshot test_target1/test_source1/fs1@compatible2") - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): #should be ok, is compatible self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) @@ -542,19 +539,19 @@ test_target1/test_source2/fs2/sub@test-20101111000000 canmount - - r=shelltest("zfs snapshot test_target1/test_source1/fs1@incompatible1") - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): #--test should fail, now incompatible self.assertTrue(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --test".split(" ")).run()) - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): #should fail, now incompatible self.assertTrue(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - with patch('time.strftime', return_value="test-20101111000003"): + with mocktime("20101111000003"): #--test should succeed by destroying incompatibles self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --destroy-incompatible --test".split(" ")).run()) - with patch('time.strftime', return_value="test-20101111000003"): + with mocktime("20101111000003"): #should succeed by destroying incompatibles self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --destroy-incompatible".split(" ")).run()) @@ -592,13 +589,13 @@ test_target1/test_source2/fs2/sub@test-20101111000003 #test all ssh directions - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --ssh-source localhost --exclude-received".split(" ")).run()) - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --ssh-target localhost --exclude-received".split(" ")).run()) - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --ssh-source localhost --ssh-target localhost".split(" ")).run()) @@ -643,7 +640,7 @@ test_target1/test_source2/fs2/sub@test-20101111000002 def test_minchange(self): #initial - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --min-change 100000".split(" ")).run()) #make small change, use umount to reflect the changes immediately @@ -653,7 +650,7 @@ test_target1/test_source2/fs2/sub@test-20101111000002 #too small change, takes no snapshots - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --min-change 100000".split(" ")).run()) #make big change @@ -661,7 +658,7 @@ test_target1/test_source2/fs2/sub@test-20101111000002 r=shelltest("zfs umount test_source1/fs1; zfs mount test_source1/fs1") #bigger change, should take snapshot - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --min-change 100000".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -694,7 +691,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 def test_test(self): #initial - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --test".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -711,12 +708,12 @@ test_target1 """) #actual make initial backup - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) #test incremental - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --allow-empty --verbose --test".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -752,7 +749,7 @@ test_target1/test_source2/fs2/sub@test-20101111000001 shelltest("zfs create test_target1/test_source1") shelltest("zfs send test_source1/fs1@migrate1| zfs recv test_target1/test_source1/fs1") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -785,15 +782,15 @@ test_target1/test_source2/fs2/sub@test-20101111000000 def test_keep0(self): """test if keep-source=0 and keep-target=0 dont delete common snapshot and break backup""" - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --keep-source=0 --keep-target=0".split(" ")).run()) #make snapshot, shouldnt delete 0 - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test --no-progress --verbose --keep-source=0 --keep-target=0 --allow-empty".split(" ")).run()) #make snapshot 2, shouldnt delete 0 since it has holds, but will delete 1 since it has no holds - with patch('time.strftime', return_value="test-20101111000002"): + with mocktime("20101111000002"): self.assertFalse(ZfsAutobackup("test --no-progress --verbose --keep-source=0 --keep-target=0 --allow-empty".split(" ")).run()) r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) @@ -825,7 +822,7 @@ test_target1/test_source2/fs2/sub@test-20101111000000 """) #make another backup but with no-holds. we should naturally endup with only number 3 - with patch('time.strftime', return_value="test-20101111000003"): + with mocktime("20101111000003"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --keep-source=0 --keep-target=0 --no-holds --allow-empty".split(" ")).run()) r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) @@ -855,7 +852,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 # run with snapshot-only for 4, since we used no-holds, it will delete 3 on the source, breaking the backup - with patch('time.strftime', return_value="test-20101111000004"): + with mocktime("20101111000004"): self.assertFalse(ZfsAutobackup("test --no-progress --verbose --keep-source=0 --keep-target=0 --allow-empty".split(" ")).run()) r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) diff --git a/tests/test_zfsautobackup31.py b/tests/test_zfsautobackup31.py index 7cd8727..17de12f 100644 --- a/tests/test_zfsautobackup31.py +++ b/tests/test_zfsautobackup31.py @@ -10,10 +10,10 @@ class TestZfsAutobackup31(unittest2.TestCase): def test_no_thinning(self): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --keep-target=0 --keep-source=0 --no-thinning".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t all "+TEST_POOLS) @@ -54,10 +54,10 @@ test_target1/test_source2/fs2/sub@test-20101111000001 shelltest("zfs create test_target1/a") shelltest("zfs create test_target1/b") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1/a --no-progress --verbose --debug".split(" ")).run()) - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup("test test_target1/b --no-progress --verbose".split(" ")).run()) r=shelltest("zfs list -H -o name -r -t snapshot test_target1") @@ -75,7 +75,7 @@ test_target1/b/test_target1/a/test_source1/fs1/sub@test-20101111000000 def test_zfs_compressed(self): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse( ZfsAutobackup("test test_target1 --no-progress --verbose --debug --zfs-compressed".split(" ")).run()) @@ -84,7 +84,7 @@ test_target1/b/test_target1/a/test_source1/fs1/sub@test-20101111000000 shelltest("zfs set autobackup:test=true test_source1") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse( ZfsAutobackup("test test_target1 --no-progress --verbose --debug --force --strip-path=1".split(" ")).run()) @@ -101,13 +101,13 @@ test_target1/fs2/sub@test-20101111000000 shelltest("zfs snapshot -r test_source1@somesnapshot") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse( ZfsAutobackup( "test test_target1 --verbose --allow-empty --exclude-unchanged=1".split(" ")).run()) #everything should be excluded, but should not return an error (see #190) - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): self.assertFalse( ZfsAutobackup( "test test_target1 --verbose --allow-empty --exclude-unchanged=1".split(" ")).run()) diff --git a/tests/test_zfsautobackup32.py b/tests/test_zfsautobackup32.py index 9b59923..e96cc32 100644 --- a/tests/test_zfsautobackup32.py +++ b/tests/test_zfsautobackup32.py @@ -9,14 +9,14 @@ class TestZfsAutobackup32(unittest2.TestCase): def test_invalid_common_snapshot(self): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) #create 2 snapshots with the same name, which are invalid as common snapshot shelltest("zfs snapshot test_source1/fs1@invalid") shelltest("zfs snapshot test_target1/test_source1/fs1@invalid") - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): #try the old way (without guid checking), and fail: self.assertEqual(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --no-guid-check".split(" ")).run(),1) #new way should be ok: @@ -57,7 +57,7 @@ test_target1/test_source2/fs2/sub@test-20101111000001 def test_invalid_common_snapshot_with_data(self): - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) #create 2 snapshots with the same name, which are invalid as common snapshot @@ -65,7 +65,7 @@ test_target1/test_source2/fs2/sub@test-20101111000001 shelltest("touch /test_target1/test_source1/fs1/shouldnotbeHere") shelltest("zfs snapshot test_target1/test_source1/fs1@invalid") - with patch('time.strftime', return_value="test-20101111000001"): + with mocktime("20101111000001"): #try the old way and fail: self.assertEqual(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --destroy-incompatible --no-guid-check".split(" ")).run(),1) #new way should be ok @@ -110,7 +110,7 @@ test_target1/test_source2/fs2/sub@test-20101111000001 shelltest("zfs create -V 10M test_source1/fs1/subvol") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty".split(" ")).run()) r=shelltest("zfs mount |grep -o /test_target1.*") @@ -127,7 +127,7 @@ test_target1/test_source2/fs2/sub@test-20101111000001 shelltest("zfs create -V 10M test_source1/fs1/subvol") - with patch('time.strftime', return_value="test-20101111000000"): + with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --clear-mountpoint".split(" ")).run()) r=shelltest("zfs mount |grep -o /test_target1.*") @@ -149,13 +149,13 @@ test_target1/test_source2/fs2/sub@test-20101111000001 print(datetime_now(False)) self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint --verbose".split(" ")).run()) -# with patch('time.strftime', return_value="test-20001101000000"): +# with mocktime("20001101000000"): # self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint test_target1 --no-progress --allow-empty --clear-mountpoint".split(" ")).run()) # -# with patch('time.strftime', return_value="test-20001201000000"): +# with mocktime("20001201000000"): # self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint".split(" ")).run()) # -# with patch('time.strftime', return_value="test-20001202000000"): +# with mocktime("20001202000000"): # self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint".split(" ")).run()) # # time_str="test-20001203000000" diff --git a/tests/test_zfscheck.py b/tests/test_zfscheck.py index 8a24973..cb239f3 100644 --- a/tests/test_zfscheck.py +++ b/tests/test_zfscheck.py @@ -1,3 +1,5 @@ +from os.path import exists + from basetest import * from zfs_autobackup.BlockHasher import BlockHasher @@ -9,6 +11,10 @@ class TestZfsCheck(unittest2.TestCase): def test_volume(self): + + if exists("/.dockerenv"): + self.skipTest("FIXME: zfscheck volumes not supported in docker yet") + prepare_zpools() shelltest("zfs create -V200M test_source1/vol") @@ -50,7 +56,7 @@ class TestZfsCheck(unittest2.TestCase): shelltest("mkfifo /test_source1/f") shelltest("zfs snapshot test_source1@test") - + ZfsCheck("test_source1@test --debug".split(" "), print_arguments=False).run() with self.subTest("Generate"): with OutputIO() as buf: with redirect_stdout(buf): @@ -178,15 +184,16 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bd8 shelltest("cp tests/data/whole /test_source1/testfile") shelltest("zfs snapshot test_source1@test") - #breaks pipe when grep exists: + #breaks pipe when head exists #important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling - shelltest("python -m zfs_autobackup.ZfsCheck test_source1@test --debug | grep -m1 'Hashing tree'") - # time.sleep(5) + shelltest("python -m zfs_autobackup.ZfsCheck test_source1@test --debug | head -n1") #should NOT be mounted anymore if cleanup went ok: self.assertNotRegex(shelltest("mount"), "test_source1@test") def test_brokenpipe_cleanup_volume(self): + if exists("/.dockerenv"): + self.skipTest("FIXME: zfscheck volumes not supported in docker yet") prepare_zpools() shelltest("zfs create -V200M test_source1/vol") @@ -194,7 +201,7 @@ whole_whole2_partial 0 309ffffba2e1977d12f3b7469971f30d28b94bd8 #breaks pipe when grep exists: #important to use --debug, since that generates extra output which would be problematic if we didnt do correct SIGPIPE handling - shelltest("python -m zfs_autobackup.ZfsCheck test_source1/vol@test --debug | grep -m1 'Hashing file'") + shelltest("python -m zfs_autobackup.ZfsCheck test_source1/vol@test --debug| grep -m1 'Hashing file'") # time.sleep(1) r = shelltest("zfs list -H -o name -r -t all " + TEST_POOLS) diff --git a/tests/tests_docker b/tests/tests_docker new file mode 100755 index 0000000..4f44278 --- /dev/null +++ b/tests/tests_docker @@ -0,0 +1,27 @@ +#!/bin/sh + +#NOTE: This script will started inside the test docker container + +set -e + +if ! [ -e /dev/ram0 ]; then + echo "Please run this outside container:" >&2 + echo "sudo modprobe brd rd_size=512000" >&2 + exit 1 + +fi + +#start sshd and other stuff +ssh-keygen -A +/usr/sbin/sshd +udevd -d + +#test ssh localhost +if ! [ -e /root/.ssh/id_rsa ]; then + ssh-keygen -t rsa -f /root/.ssh/id_rsa -P '' +fi +cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys +ssh -oStrictHostKeyChecking=no localhost 'echo SSH OK' + +cd /app +python -m unittest discover /app/tests -vvvvf $@ diff --git a/zfs_autobackup/ZfsAutoverify.py b/zfs_autobackup/ZfsAutoverify.py index 3534a5e..f8c27ae 100644 --- a/zfs_autobackup/ZfsAutoverify.py +++ b/zfs_autobackup/ZfsAutoverify.py @@ -86,8 +86,8 @@ def verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt, raise(Exception("program errror, unknown method")) finally: - source_snapshot.unmount() - target_snapshot.unmount() + source_snapshot.unmount(source_mnt) + target_snapshot.unmount(target_mnt) # def hash_dev(node, dev): diff --git a/zfs_autobackup/ZfsCheck.py b/zfs_autobackup/ZfsCheck.py index f0a0cab..10fe8ae 100644 --- a/zfs_autobackup/ZfsCheck.py +++ b/zfs_autobackup/ZfsCheck.py @@ -74,7 +74,7 @@ class ZfsCheck(CliBase): def cleanup_zfs_filesystem(self, snapshot): mnt = "/tmp/" + tmp_name() - snapshot.unmount() + snapshot.unmount(mnt) self.debug("Cleaning up temporary mount point") self.node.run(["rmdir", mnt], hide_errors=True, valid_exitcodes=[]) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index d659cc2..985afee 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1230,12 +1230,12 @@ class ZfsDataset: self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) - def unmount(self): + def unmount(self, mount_point): self.debug("Unmounting") cmd = [ - "umount", self.name + "umount", mount_point ] self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index 560aa62..982b20f 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -61,7 +61,7 @@ class ZfsNode(ExecuteNode): # NOTE: if thinning is disabled with --no-thinning, self.__thinner will be none. if self.__thinner is not None: - return self.__thinner.thin(objects, keep_objects, datetime_now(self.utc)) + return self.__thinner.thin(objects, keep_objects, datetime_now(self.utc).timestamp()) else: return (keep_objects, [])