From 176f04b3021a32b931f690a5018e838527bbbc4c Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Tue, 20 Apr 2021 23:20:54 +0200 Subject: [PATCH] proper encryption/decryption support. also fixes #60 --- README.md | 14 ++++- tests/test_encryption.py | 98 ++++++++++++++++++++++++++++----- tests/test_zfsautobackup.py | 2 +- zfs_autobackup/ZfsAutobackup.py | 2 +- zfs_autobackup/ZfsDataset.py | 43 +++++++++------ 5 files changed, 123 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 7930c01..68403e2 100644 --- a/README.md +++ b/README.md @@ -375,16 +375,24 @@ Snapshots on the source that still have to be send to the target wont be destroy In normal operation datasets are transferred unaltered: * Source datasets that are encrypted will be send over as such and stay encrypted at the target side. (In ZFS this is called raw-mode) You dont need keys at the target side if you dont want to access the data. -* Source datasets that are plain will stay that way on the target. Even if the specified target-path IS encrypted. +* Source datasets that are plain will stay that way on the target. (Even if the specified target-path IS encrypted.) + +Basically you dont have to do anything or worry about anything. ### Decrypting/encrypting -If you want to alter the encryption-state of a dataset you have several options: +Things get different if you want to change the encryption-state of a dataset during transfer: * If you want to decrypt encrypted datasets before sending them, you should use the `--decrypt` option. Datasets will then be stored plain at the target. -* If you want to encrypt plain datasets when they are received, you should use the `--encrypt` option. Datasets will then be stored encrypted at the target. (Datasets that are already encrypted will still be sent over unaltered!) You are responsible for creating the target-path with encryption enabled. +* If you want to encrypt plain datasets when they are received, you should use the `--encrypt` option. Datasets will then be stored encrypted at the target. (Datasets that are already encrypted will still be sent over unaltered!) * If you also want re-encrypt encrypted datasets with the target-side encryption you can use both options. +Note 1: The --encrypt option will rely on inheriting encryption parameters from the parent datasets on the parent side. You are responsible for setting those up and loading the keys. So --encrypt is no guarantee for encryption, if its not setup, it cant be encrypted. + +Note 2: Decide what you want at an early stage: If you change the --encrypt or --decrypt parameter at a later time you might get weird and wonderfull errors. (nothing dangerous) + +I'll add some tips when the issues start to get in on github. :) + ## Tips * Use ```--debug``` if something goes wrong and you want to see the commands that are executed. This will also stop at the first error. diff --git a/tests/test_encryption.py b/tests/test_encryption.py index ce4f4d8..b5a4090 100644 --- a/tests/test_encryption.py +++ b/tests/test_encryption.py @@ -2,6 +2,21 @@ from zfs_autobackup.CmdPipe import CmdPipe from basetest import * import time +# We have to do a LOT to properly test encryption/decryption/raw transfers +# +# For every scenario we need at least: +# - plain source dataset +# - encrypted source dataset +# - plain target path +# - encrypted target path +# - do a full transfer +# - do a incremental transfer + +# Scenarios: +# - Raw transfer +# - Decryption transfer (--decrypt) +# - Encryption transfer (--encrypt) +# - Re-encryption transfer (--decrypt --encrypt) class TestZfsEncryption(unittest2.TestCase): @@ -65,8 +80,12 @@ test_target1/test_source2/fs2/sub encryption self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") with patch('time.strftime', return_value="20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot".split(" ")).run()) + + with patch('time.strftime', return_value="20101111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot".split(" ")).run()) r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") self.assertEqual(r, """ @@ -96,21 +115,72 @@ test_target1/test_source2/fs2/sub encryptionroot - self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") with patch('time.strftime', return_value="20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt".split(" ")).run()) - self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot".split(" ")).run()) - r = shelltest("zfs get encryption -H -o value test_target1/test_source1/fs1/encryptedsource test_target1/encryptedtarget/test_source1/fs1/encryptedsource") - self.assertNotIn("off",r) + with patch('time.strftime', return_value="20101111000001"): + self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot".split(" ")).run()) + r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") + self.assertEqual(r, """ +NAME PROPERTY VALUE SOURCE +test_target1 encryptionroot - - +test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget - +test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget - +test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget - +test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource - +test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget - +test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget - +test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget - +test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget - +test_target1/test_source1 encryptionroot - - +test_target1/test_source1/fs1 encryptionroot - - +test_target1/test_source1/fs1/encryptedsource encryptionroot test_target1/test_source1/fs1/encryptedsource - +test_target1/test_source1/fs1/sub encryptionroot - - +test_target1/test_source2 encryptionroot - - +test_target1/test_source2/fs2 encryptionroot - - +test_target1/test_source2/fs2/sub encryptionroot - - +""") - def test_reencrypt(self): - """decrypt data and reencrypt on the otherside (--decrypt --encrypt) """ + def test_reencrypt(self): + """reencrypt data (--decrypt --encrypt) """ - # create encrypted target dataset - shelltest("echo 12345678 > /tmp/zfstest.key") - shelltest("zfs create -o keylocation=file:///tmp/zfstest.key -o keyformat=passphrase -o encryption=on test_target1/enc1") + self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource") + self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") with patch('time.strftime', return_value="20101111000000"): - self.assertFalse(ZfsAutobackup("test test_target1/enc1 --allow-empty --verbose --no-progress".split(" ")).run()) - r = shelltest("zfs get encryption -H -o value test_target1/enc1/test_source1/fs1") - self.assertNotIn("off",r) + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot".split( + " ")).run()) + + with patch('time.strftime', return_value="20101111000001"): + self.assertFalse(ZfsAutobackup( + "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty".split(" ")).run()) + self.assertFalse(ZfsAutobackup( + "test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot".split( + " ")).run()) + + r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") + self.assertEqual(r, """ +NAME PROPERTY VALUE SOURCE +test_target1 encryptionroot - - +test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget - +test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget - +test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget - +test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget - +test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget - +test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget - +test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget - +test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget - +test_target1/test_source1 encryptionroot - - +test_target1/test_source1/fs1 encryptionroot - - +test_target1/test_source1/fs1/encryptedsource encryptionroot - - +test_target1/test_source1/fs1/sub encryptionroot - - +test_target1/test_source2 encryptionroot - - +test_target1/test_source2/fs2 encryptionroot - - +test_target1/test_source2/fs2/sub encryptionroot - - +""") + diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 587d576..44fb02b 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -890,7 +890,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 n=ZfsNode("test",l) d=ZfsDataset(n,"test_source1@test") - sp=d.send_pipe([], prev_snapshot=None, resume_token=None, show_progress=True, raw=False, output_pipes=[], send_properties=False) + sp=d.send_pipe([], prev_snapshot=None, resume_token=None, show_progress=True, raw=False, output_pipes=[], send_properties=True, write_embedded=True) with OutputIO() as buf: diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index d353a48..8c3263e 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -268,7 +268,7 @@ class ZfsAutobackup: also_other_snapshots=self.args.other_snapshots, no_send=self.args.no_send, destroy_incompatible=self.args.destroy_incompatible, - output_pipes=self.args.send_pipe, input_pipes=self.args.recv_pipe, decrypt=self.args.decrypt) + output_pipes=self.args.send_pipe, input_pipes=self.args.recv_pipe, decrypt=self.args.decrypt, encrypt=self.args.encrypt) except Exception as e: fail_count = fail_count + 1 source_dataset.error("FAILED: " + str(e)) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 8c892fd..eb60f79 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -494,7 +494,7 @@ class ZfsDataset: return self.from_names(names[1:]) - def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, output_pipes): + def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, write_embedded, output_pipes): """returns a pipe with zfs send output for this snapshot resume_token: resume sending from this token. (in that case we don't @@ -515,13 +515,13 @@ class ZfsDataset: # all kind of performance options: if 'large_blocks' in features and "-L" in self.zfs_node.supported_send_options: - cmd.append("-L") # large block support (only if recordsize>128k which is seldomly used) + cmd.append("--large-block") # large block support (only if recordsize>128k which is seldomly used) - if 'embedded_data' in features and "-e" in self.zfs_node.supported_send_options: - cmd.append("-e") # WRITE_EMBEDDED, more compact stream + if write_embedded and 'embedded_data' in features and "-e" in self.zfs_node.supported_send_options: + cmd.append("--embed") # WRITE_EMBEDDED, more compact stream if "-c" in self.zfs_node.supported_send_options: - cmd.append("-c") # use compressed WRITE records + cmd.append("--compressed") # use compressed WRITE records # raw? (send over encrypted data in its original encrypted form without decrypting) if raw: @@ -529,8 +529,8 @@ class ZfsDataset: # progress output if show_progress: - cmd.append("-v") - cmd.append("-P") + cmd.append("--verbose") + cmd.append("--parsable") # resume a previous send? (don't need more parameters in that case) if resume_token: @@ -539,7 +539,7 @@ class ZfsDataset: else: # send properties if send_properties: - cmd.append("-p") + cmd.append("--props") # incremental? if prev_snapshot: @@ -632,7 +632,7 @@ class ZfsDataset: def transfer_snapshot(self, target_snapshot, features, prev_snapshot, show_progress, filter_properties, set_properties, ignore_recv_exit_code, resume_token, - raw, send_properties, output_pipes, input_pipes): + raw, send_properties, write_embedded, output_pipes, input_pipes): """transfer this snapshot to target_snapshot. specify prev_snapshot for incremental transfer @@ -671,7 +671,7 @@ class ZfsDataset: # do it pipe = self.send_pipe(features=features, show_progress=show_progress, prev_snapshot=prev_snapshot, - resume_token=resume_token, raw=raw, send_properties=send_properties, output_pipes=output_pipes) + resume_token=resume_token, raw=raw, send_properties=send_properties, write_embedded=write_embedded, output_pipes=output_pipes) target_snapshot.recv_pipe(pipe, features=features, filter_properties=filter_properties, set_properties=set_properties, ignore_exit_code=ignore_recv_exit_code) @@ -960,7 +960,7 @@ class ZfsDataset: def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, - ignore_recv_exit_code, holds, rollback, decrypt, also_other_snapshots, + ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots, no_send, destroy_incompatible, output_pipes, input_pipes): """sync this dataset's snapshots to target_dataset, while also thinning out old snapshots along the way. @@ -1007,8 +1007,12 @@ class ZfsDataset: if rollback: target_dataset.rollback() + #defaults for these settings if there is no encryption stuff going on: send_properties = True raw = False + write_embedded = True + + (active_filter_properties, active_set_properties) = self.get_allowed_properties(filter_properties, set_properties) # source dataset encrypted? if self.properties.get('encryption', 'off')!='off': @@ -1020,6 +1024,13 @@ class ZfsDataset: # keep data encrypted by sending it raw (including properties) raw=True + # encrypt at target? + if encrypt and not raw: + # filter out encryption properties to let encryption on the target take place + active_filter_properties.extend(["keylocation","pbkdf2iters","keyformat", "encryption"]) + write_embedded=False + + # now actually transfer the snapshots prev_source_snapshot = common_snapshot source_snapshot = start_snapshot @@ -1028,15 +1039,13 @@ class ZfsDataset: # does target actually want it? if target_snapshot not in target_obsoletes: - # NOTE: should we let transfer_snapshot handle this? - (allowed_filter_properties, allowed_set_properties) = self.get_allowed_properties(filter_properties, - set_properties) + source_snapshot.transfer_snapshot(target_snapshot, features=features, prev_snapshot=prev_source_snapshot, show_progress=show_progress, - filter_properties=allowed_filter_properties, - set_properties=allowed_set_properties, + filter_properties=active_filter_properties, + set_properties=active_set_properties, ignore_recv_exit_code=ignore_recv_exit_code, - resume_token=resume_token, raw=raw, send_properties=send_properties, output_pipes=output_pipes, input_pipes=input_pipes) + resume_token=resume_token, write_embedded=write_embedded,raw=raw, send_properties=send_properties, output_pipes=output_pipes, input_pipes=input_pipes) resume_token = None