This commit is contained in:
Edwin Eefting 2024-10-08 16:51:03 +02:00
parent 97123502b0
commit d97a5f9527
No known key found for this signature in database
GPG Key ID: 0F3C35D8E9887737
2 changed files with 91 additions and 72 deletions

View File

@ -2,6 +2,7 @@ from zfs_autobackup.CmdPipe import CmdPipe
from basetest import * from basetest import *
import time import time
# We have to do a LOT to properly test encryption/decryption/raw transfers # We have to do a LOT to properly test encryption/decryption/raw transfers
# #
# For every scenario we need at least: # For every scenario we need at least:
@ -20,7 +21,6 @@ import time
class TestZfsEncryption(unittest2.TestCase): class TestZfsEncryption(unittest2.TestCase):
def setUp(self): def setUp(self):
prepare_zpools() prepare_zpools()
@ -40,7 +40,9 @@ class TestZfsEncryption(unittest2.TestCase):
# create encrypted source dataset # create encrypted source dataset
shelltest("rm /tmp/zfstest.key 2>/dev/null;true") shelltest("rm /tmp/zfstest.key 2>/dev/null;true")
shelltest("echo {} > /tmp/zfstest.key".format(key)) shelltest("echo {} > /tmp/zfstest.key".format(key))
shelltest("zfs create -o keylocation=file:///tmp/zfstest.key -o keyformat=passphrase -o encryption=on {}".format(path)) shelltest(
"zfs create -o keylocation=file:///tmp/zfstest.key -o keyformat=passphrase -o encryption=on {}".format(
path))
if unload_key: if unload_key:
shelltest("zfs unmount {}".format(path)) shelltest("zfs unmount {}".format(path))
@ -48,23 +50,30 @@ class TestZfsEncryption(unittest2.TestCase):
# r=shelltest("dd if=/dev/zero of=/test_source1/fs1/enc1/data.txt bs=200000 count=1") # r=shelltest("dd if=/dev/zero of=/test_source1/fs1/enc1/data.txt bs=200000 count=1")
def test_raw(self): def test_raw(self):
"""send encrypted data unaltered (standard operation)""" """send encrypted data unaltered (standard operation)"""
self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource")
self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsourcekeyless", unload_key=True) # raw mode shouldn't need a key self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsourcekeyless",
unload_key=True) # raw mode shouldn't need a key
self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget")
with mocktime("20101111000000"): with mocktime("20101111000000"):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) self.assertFalse(ZfsAutobackup(
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split(" ")).run()) "test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run())
self.assertFalse(ZfsAutobackup(
"test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split(
" ")).run())
with mocktime("20101111000001"): with mocktime("20101111000001"):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run()) self.assertFalse(ZfsAutobackup(
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split(" ")).run()) "test test_target1 --verbose --no-progress --allow-empty --exclude-received".split(" ")).run())
self.assertFalse(ZfsAutobackup(
"test test_target1/encryptedtarget --verbose --no-progress --no-snapshot --exclude-received".split(
" ")).run())
r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") r = shelltest("zfs get -r -t filesystem encryptionroot test_target1")
self.assertMultiLineEqual(r,""" self.assertMultiLineEqual(r, """
NAME PROPERTY VALUE SOURCE NAME PROPERTY VALUE SOURCE
test_target1 encryptionroot - - test_target1 encryptionroot - -
test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget - test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget -
@ -86,22 +95,28 @@ test_target1/test_source2/fs2 encryption
test_target1/test_source2/fs2/sub encryptionroot - - test_target1/test_source2/fs2/sub encryptionroot - -
""") """)
def test_decrypt(self): def test_decrypt(self):
"""decrypt data and store unencrypted (--decrypt)""" """decrypt data and store unencrypted (--decrypt)"""
self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource")
self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget")
with mocktime("20101111000000"): with mocktime("20101111000000"):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(" ")).run()) self.assertFalse(ZfsAutobackup(
"test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(
" ")).run())
# NOTE: this also tests if sending a source to a second target works correctly (this failed during bookmark implementation) # NOTE: this also tests if sending a source to a second target works correctly (this failed during bookmark implementation)
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received --debug".split(" ")).run()) self.assertFalse(ZfsAutobackup(
"test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received --debug".split(
" ")).run())
with mocktime("20101111000001"): with mocktime("20101111000001"):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(" ")).run()) self.assertFalse(ZfsAutobackup(
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received".split(" ")).run()) "test test_target1 --verbose --no-progress --decrypt --allow-empty --exclude-received".split(
" ")).run())
self.assertFalse(ZfsAutobackup(
"test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot --exclude-received".split(
" ")).run())
r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") r = shelltest("zfs get -r -t filesystem encryptionroot test_target1")
self.assertEqual(r, """ self.assertEqual(r, """
@ -124,19 +139,27 @@ test_target1/test_source2/fs2 encryptionroot -
test_target1/test_source2/fs2/sub encryptionroot - - test_target1/test_source2/fs2/sub encryptionroot - -
""") """)
def test_encrypt(self): def test_encrypt(self):
"""send normal data set and store encrypted on the other side (--encrypt) issue #60 """ """send normal data set and store encrypted on the other side (--encrypt) issue #60 """
self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource")
self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget")
with mocktime("20101111000000"): with mocktime("20101111000000"):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) self.assertFalse(ZfsAutobackup(
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split(" ")).run()) "test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(
" ")).run())
self.assertFalse(ZfsAutobackup(
"test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split(
" ")).run())
with mocktime("20101111000001"): with mocktime("20101111000001"):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) self.assertFalse(ZfsAutobackup(
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split(" ")).run()) "test test_target1 --verbose --no-progress --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(
" ")).run())
self.assertFalse(ZfsAutobackup(
"test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split(
" ")).run())
r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") r = shelltest("zfs get -r -t filesystem encryptionroot test_target1")
self.assertEqual(r, """ self.assertEqual(r, """
@ -167,14 +190,16 @@ test_target1/test_source2/fs2/sub encryptionroot -
with mocktime("20101111000000"): with mocktime("20101111000000"):
self.assertFalse(ZfsAutobackup( self.assertFalse(ZfsAutobackup(
"test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received --clear-mountpoint".split(
" ")).run())
self.assertFalse(ZfsAutobackup( self.assertFalse(ZfsAutobackup(
"test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split( "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --exclude-received --clear-mountpoint".split(
" ")).run()) " ")).run())
with mocktime("20101111000001"): with mocktime("20101111000001"):
self.assertFalse(ZfsAutobackup( self.assertFalse(ZfsAutobackup(
"test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received".split(" ")).run()) "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty --exclude-received".split(
" ")).run())
self.assertFalse(ZfsAutobackup( self.assertFalse(ZfsAutobackup(
"test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --exclude-received".split( "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot --exclude-received".split(
" ")).run()) " ")).run())
@ -200,10 +225,7 @@ test_target1/test_source2/fs2 encryptionroot -
test_target1/test_source2/fs2/sub encryptionroot - - test_target1/test_source2/fs2/sub encryptionroot - -
""") """)
def test_raw_invalid_snapshot(self):
def test_raw_invalid_snapshot(self):
"""in raw mode, its not allowed to have any newer snaphots on target, #219""" """in raw mode, its not allowed to have any newer snaphots on target, #219"""
self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource")
@ -211,18 +233,19 @@ test_target1/test_source2/fs2/sub encryptionroot -
with mocktime("20101111000000"): with mocktime("20101111000000"):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress".split(" ")).run())
#this is invalid in raw mode # this is invalid in raw mode
shelltest("zfs snapshot test_target1/test_source1/fs1/encryptedsource@incompatible") shelltest("zfs snapshot test_target1/test_source1/fs1/encryptedsource@incompatible")
with mocktime("20101111000001"): with mocktime("20101111000001"):
#should fail because of incompatble snapshot # should fail because of incompatble snapshot
self.assertEqual(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty".split(" ")).run(),1) self.assertEqual(ZfsAutobackup("test test_target1 --verbose --no-progress --allow-empty".split(" ")).run(),
#should destroy incompatible and continue 1)
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --no-snapshot --destroy-incompatible".split(" ")).run()) # should destroy incompatible and continue
self.assertFalse(ZfsAutobackup(
"test test_target1 --verbose --no-progress --no-snapshot --destroy-incompatible".split(" ")).run())
r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") r = shelltest("zfs get -r -t filesystem encryptionroot test_target1")
self.assertMultiLineEqual(r,""" self.assertMultiLineEqual(r, """
NAME PROPERTY VALUE SOURCE NAME PROPERTY VALUE SOURCE
test_target1 encryptionroot - - test_target1 encryptionroot - -
test_target1/test_source1 encryptionroot - - test_target1/test_source1 encryptionroot - -
@ -234,16 +257,16 @@ test_target1/test_source2/fs2 encryptionroot -
test_target1/test_source2/fs2/sub encryptionroot - - test_target1/test_source2/fs2/sub encryptionroot - -
""") """)
def test_resume_encrypt_with_no_key(self):
def test_resume_encrypt_with_no_key(self):
"""test what happens if target encryption key not loaded (this led to a kernel crash of freebsd with 2.1.x i think) while trying to resume""" """test what happens if target encryption key not loaded (this led to a kernel crash of freebsd with 2.1.x i think) while trying to resume"""
self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource")
self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget")
with mocktime("20101111000000"): with mocktime("20101111000000"):
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --exclude-received --clear-mountpoint".split(" ")).run()) self.assertFalse(ZfsAutobackup(
"test test_target1/encryptedtarget --verbose --no-progress --encrypt --allow-empty --exclude-received --clear-mountpoint".split(
" ")).run())
r = shelltest("zfs set compress=off test_source1 test_target1") r = shelltest("zfs set compress=off test_source1 test_target1")
@ -274,43 +297,38 @@ test_target1/test_source2/fs2/sub encryptionroot -
with mocktime("20101111000001"): with mocktime("20101111000001"):
self.assertEqual(ZfsAutobackup( self.assertEqual(ZfsAutobackup(
"test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint".split( "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint".split(
" ")).run(),3) " ")).run(), 3)
# NOTE: On some versions this leaves 2 weird sub-datasets that should'nt be there (its probably a zfs bug?)
# so we ignore this, and just make sure the backup resumes correctly after reloading the key.
# r = shelltest("zfs get -r -t all encryptionroot test_target1")
# self.assertEqual(r, """
# NAME PROPERTY VALUE SOURCE
# test_target1 encryptionroot - -
# test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source1/fs1@test-20101111000000 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource -
# test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000000 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource -
# test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000001 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource -
# test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source1/fs1/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source1/fs1/sub/sub encryptionroot - -
# test_target1/encryptedtarget/test_source1/fs1/sub/sub@test-20101111000001 encryptionroot - -
# test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source2/fs2/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source2/fs2/sub/sub encryptionroot - -
# test_target1/encryptedtarget/test_source2/fs2/sub/sub@test-20101111000001 encryptionroot - -
# """)
# reload key and resume correctly.
#NOTE: On some versions this leaves 2 weird sub-datasets that should'nt be there (its probably a zfs bug?)
#so we ignore this, and just make sure the backup resumes correctly after reloading the key.
# r = shelltest("zfs get -r -t all encryptionroot test_target1")
# self.assertEqual(r, """
# NAME PROPERTY VALUE SOURCE
# test_target1 encryptionroot - -
# test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source1/fs1@test-20101111000000 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource -
# test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000000 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource -
# test_target1/encryptedtarget/test_source1/fs1/encryptedsource@test-20101111000001 encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource -
# test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source1/fs1/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source1/fs1/sub/sub encryptionroot - -
# test_target1/encryptedtarget/test_source1/fs1/sub/sub@test-20101111000001 encryptionroot - -
# test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source2/fs2/sub@test-20101111000000 encryptionroot test_target1/encryptedtarget -
# test_target1/encryptedtarget/test_source2/fs2/sub/sub encryptionroot - -
# test_target1/encryptedtarget/test_source2/fs2/sub/sub@test-20101111000001 encryptionroot - -
# """)
#reload key and resume correctly.
self.load_key("22222222", "test_target1/encryptedtarget") self.load_key("22222222", "test_target1/encryptedtarget")
# resume should complete # resume should complete
with mocktime("20101111000001"): with mocktime("20101111000001"):
self.assertEqual(ZfsAutobackup( self.assertEqual(ZfsAutobackup(
"test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint".split( "test test_target1/encryptedtarget --verbose --no-progress --encrypt --exclude-received --allow-empty --no-snapshot --clear-mountpoint --debug".split(
" ")).run(),0) " ")).run(), 0)

View File

@ -984,6 +984,7 @@ class ZfsDataset:
Args: Args:
:type resume_token: str :type resume_token: str
:rtype: ZfsDataset|None
""" """
# use zfs send -n option to determine this # use zfs send -n option to determine this
# NOTE: on smartos stderr, on linux stdout # NOTE: on smartos stderr, on linux stdout
@ -1170,7 +1171,7 @@ class ZfsDataset:
resume_token = target_dataset.properties['receive_resume_token'] resume_token = target_dataset.properties['receive_resume_token']
# not valid anymore # not valid anymore
resume_snapshot = self.get_resume_snapshot(resume_token) resume_snapshot = self.get_resume_snapshot(resume_token)
if not resume_snapshot or start_snapshot.suffix != resume_snapshot.snapshot_name: if not resume_snapshot or start_snapshot.suffix != resume_snapshot.suffix:
target_dataset.verbose("Aborting resume, its no longer valid.") target_dataset.verbose("Aborting resume, its no longer valid.")
target_dataset.abort_resume() target_dataset.abort_resume()
else: else: