From d97a5f952741f81f23e725ba977ebe21014da0a7 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 8 Oct 2024 16:51:03 +0200 Subject: [PATCH] fix --- tests/test_encryption.py | 160 +++++++++++++++++++---------------- zfs_autobackup/ZfsDataset.py | 3 +- 2 files changed, 91 insertions(+), 72 deletions(-) diff --git a/tests/test_encryption.py b/tests/test_encryption.py index 257ade6..3193f26 100644 --- a/tests/test_encryption.py +++ b/tests/test_encryption.py @@ -2,6 +2,7 @@ from zfs_autobackup.CmdPipe import CmdPipe from basetest import * import time + # We have to do a LOT to properly test encryption/decryption/raw transfers # # For every scenario we need at least: @@ -20,7 +21,6 @@ import time class TestZfsEncryption(unittest2.TestCase): - def setUp(self): prepare_zpools() @@ -40,7 +40,9 @@ class TestZfsEncryption(unittest2.TestCase): # create encrypted source dataset shelltest("rm /tmp/zfstest.key 2>/dev/null;true") shelltest("echo {} > /tmp/zfstest.key".format(key)) - shelltest("zfs create -o keylocation=file:///tmp/zfstest.key -o keyformat=passphrase -o encryption=on {}".format(path)) + shelltest( + "zfs create -o keylocation=file:///tmp/zfstest.key -o keyformat=passphrase -o encryption=on {}".format( + path)) if unload_key: shelltest("zfs unmount {}".format(path)) @@ -48,23 +50,30 @@ class TestZfsEncryption(unittest2.TestCase): # r=shelltest("dd if=/dev/zero of=/test_source1/fs1/enc1/data.txt bs=200000 count=1") - def test_raw(self): + def test_raw(self): """send encrypted data unaltered (standard operation)""" self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") - self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsourcekeyless", unload_key=True) # raw mode shouldn't need a key + self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsourcekeyless", + unload_key=True) # raw mode shouldn't need a key self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split( + " ")).run()) with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split( + " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") - self.assertMultiLineEqual(r,""" + self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_target1 encryptionroot - - test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget - @@ -86,22 +95,28 @@ test_target1/test_source2/fs2 encryption test_target1/test_source2/fs2/sub encryptionroot - - """) - def test_decrypt(self): + def test_decrypt(self): """decrypt data and store unencrypted (--decrypt)""" - - self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split( + " ")).run()) # NOTE: this also tests if sending a source to a second target works correctly (this failed during bookmark implementation) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received --debug".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received --debug".split( + " ")).run()) with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split( + " ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received".split( + " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") self.assertEqual(r, """ @@ -124,19 +139,27 @@ test_target1/test_source2/fs2 encryptionroot - test_target1/test_source2/fs2/sub encryptionroot - - """) - def test_encrypt(self): + def test_encrypt(self): """send normal data set and store encrypted on the other side (--encrypt) issue #60 """ self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split( + " ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split( + " ")).run()) with mocktime("20101111000001"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split( + " ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split( + " ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") self.assertEqual(r, """ @@ -167,14 +190,16 @@ test_target1/test_source2/fs2/sub encryptionroot - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) + "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split( + " ")).run()) self.assertFalse(ZfsAutobackup( "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split( " ")).run()) with mocktime("20101111000001"): self.assertFalse(ZfsAutobackup( - "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received".split(" ")).run()) + "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received".split( + " ")).run()) self.assertFalse(ZfsAutobackup( "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --exclude-received".split( " ")).run()) @@ -200,10 +225,7 @@ test_target1/test_source2/fs2 encryptionroot - test_target1/test_source2/fs2/sub encryptionroot - - """) - - - - def test_raw_invalid_snapshot(self): + def test_raw_invalid_snapshot(self): """in raw mode, its not allowed to have any newer snaphots on target, #219""" self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") @@ -211,18 +233,19 @@ test_target1/test_source2/fs2/sub encryptionroot - with mocktime("20101111000000"): self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress".split(" ")).run()) - #this is invalid in raw mode + # this is invalid in raw mode shelltest("zfs snapshot test_target1/test_source1/fs1/encryptedsource@incompatible") with mocktime("20101111000001"): - #should fail because of incompatble snapshot - self.assertEqual(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty".split(" ")).run(),1) - #should destroy incompatible and continue - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --no-snapshot --destroy-incompatible".split(" ")).run()) - + # should fail because of incompatble snapshot + self.assertEqual(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty".split(" ")).run(), + 1) + # should destroy incompatible and continue + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --no-snapshot --destroy-incompatible".split(" ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") - self.assertMultiLineEqual(r,""" + self.assertMultiLineEqual(r, """ NAME PROPERTY VALUE SOURCE test_target1 encryptionroot - - test_target1/test_source1 encryptionroot - - @@ -234,16 +257,16 @@ test_target1/test_source2/fs2 encryptionroot - test_target1/test_source2/fs2/sub encryptionroot - - """) - - def test_resume_encrypt_with_no_key(self): + def test_resume_encrypt_with_no_key(self): """test what happens if target encryption key not loaded (this led to a kernel crash of freebsd with 2.1.x i think) while trying to resume""" self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") - with mocktime("20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --exclude-received --clear-mountpoint".split( + " ")).run()) r = shelltest("zfs set compress=off test_source1 test_target1") @@ -274,43 +297,38 @@ test_target1/test_source2/fs2/sub encryptionroot - with mocktime("20101111000001"): self.assertEqual(ZfsAutobackup( "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint".split( - " ")).run(),3) + " ")).run(), 3) + # NOTE: On some versions this leaves 2 weird sub-datasets that should'nt be there (its probably a zfs bug?) + # so we ignore this, and just make sure the backup resumes correctly after reloading the key. + # r = shelltest("zfs get -r -t all encryptionroot test_target1") + # self.assertEqual(r, """ + # NAME PROPERTY VALUE SOURCE + # test_target1 encryptionroot - - + # test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1@test-20101111000000 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - + # test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000000 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - + # test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000001 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - + # test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source1/fs1/sub/sub encryptionroot - - + # test_target1/encryptedtarget/test_source1/fs1/sub/sub@test-20101111000001 encryptionroot - - + # test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source2/fs2/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget - + # test_target1/encryptedtarget/test_source2/fs2/sub/sub encryptionroot - - + # test_target1/encryptedtarget/test_source2/fs2/sub/sub@test-20101111000001 encryptionroot - - + # """) - -#NOTE: On some versions this leaves 2 weird sub-datasets that should'nt be there (its probably a zfs bug?) -#so we ignore this, and just make sure the backup resumes correctly after reloading the key. -# r = shelltest("zfs get -r -t all encryptionroot test_target1") -# self.assertEqual(r, """ -# NAME PROPERTY VALUE SOURCE -# test_target1 encryptionroot - - -# test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1@test-20101111000000 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - -# test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000000 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - -# test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000001 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - -# test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source1/fs1/sub/sub encryptionroot - - -# test_target1/encryptedtarget/test_source1/fs1/sub/sub@test-20101111000001 encryptionroot - - -# test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source2/fs2/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget - -# test_target1/encryptedtarget/test_source2/fs2/sub/sub encryptionroot - - -# test_target1/encryptedtarget/test_source2/fs2/sub/sub@test-20101111000001 encryptionroot - - -# """) - - - - #reload key and resume correctly. + # reload key and resume correctly. self.load_key("22222222", "test_target1/encryptedtarget") # resume should complete with mocktime("20101111000001"): self.assertEqual(ZfsAutobackup( - "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint".split( - " ")).run(),0) - + "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint --debug".split( + " ")).run(), 0) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index bd135e2..fc6e3c7 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -984,6 +984,7 @@ class ZfsDataset: Args: :type resume_token: str + :rtype: ZfsDataset|None """ # use zfs send -n option to determine this # NOTE: on smartos stderr, on linux stdout @@ -1170,7 +1171,7 @@ class ZfsDataset: resume_token = target_dataset.properties['receive_resume_token'] # not valid anymore resume_snapshot = self.get_resume_snapshot(resume_token) - if not resume_snapshot or start_snapshot.suffix != resume_snapshot.snapshot_name: + if not resume_snapshot or start_snapshot.suffix != resume_snapshot.suffix: target_dataset.verbose("Aborting resume, its no longer valid.") target_dataset.abort_resume() else: