third-party-mirrors/zfs_autobackup
1
0
Fork 1
mirror of https://github.com/psy0rz/zfs_autobackup.git synced 2025-06-07 01:43:00 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

proper encryption/decryption support. also fixes #60

Browse Source
This commit is contained in:
Edwin Eefting 2021-04-20 23:20:54 +02:00
parent 7696d8c16d
commit 176f04b302
5 changed files with 123 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
README.md
View File

@ -375,16 +375,24 @@ Snapshots on the source that still have to be send to the target wont be destroy
In normal operation datasets are transferred unaltered: In normal operation datasets are transferred unaltered:
* Source datasets that are encrypted will be send over as such and stay encrypted at the target side. (In ZFS this is called raw-mode) You dont need keys at the target side if you dont want to access the data. * Source datasets that are encrypted will be send over as such and stay encrypted at the target side. (In ZFS this is called raw-mode) You dont need keys at the target side if you dont want to access the data.
* Source datasets that are plain will stay that way on the target. Even if the specified target-path IS encrypted. * Source datasets that are plain will stay that way on the target. (Even if the specified target-path IS encrypted.)
Basically you dont have to do anything or worry about anything.
### Decrypting/encrypting ### Decrypting/encrypting
If you want to alter the encryption-state of a dataset you have several options: Things get different if you want to change the encryption-state of a dataset during transfer:
* If you want to decrypt encrypted datasets before sending them, you should use the `--decrypt` option. Datasets will then be stored plain at the target. * If you want to decrypt encrypted datasets before sending them, you should use the `--decrypt` option. Datasets will then be stored plain at the target.
* If you want to encrypt plain datasets when they are received, you should use the `--encrypt` option. Datasets will then be stored encrypted at the target. (Datasets that are already encrypted will still be sent over unaltered!) You are responsible for creating the target-path with encryption enabled. * If you want to encrypt plain datasets when they are received, you should use the `--encrypt` option. Datasets will then be stored encrypted at the target. (Datasets that are already encrypted will still be sent over unaltered!)
* If you also want re-encrypt encrypted datasets with the target-side encryption you can use both options. * If you also want re-encrypt encrypted datasets with the target-side encryption you can use both options.
Note 1: The --encrypt option will rely on inheriting encryption parameters from the parent datasets on the parent side. You are responsible for setting those up and loading the keys. So --encrypt is no guarantee for encryption, if its not setup, it cant be encrypted.
Note 2: Decide what you want at an early stage: If you change the --encrypt or --decrypt parameter at a later time you might get weird and wonderfull errors. (nothing dangerous)
I'll add some tips when the issues start to get in on github. :)
## Tips ## Tips
* Use ```--debug``` if something goes wrong and you want to see the commands that are executed. This will also stop at the first error. * Use ```--debug``` if something goes wrong and you want to see the commands that are executed. This will also stop at the first error.

96
tests/test_encryption.py
View File

@ -2,6 +2,21 @@ from zfs_autobackup.CmdPipe import CmdPipe
from basetest import * from basetest import *
import time import time
# We have to do a LOT to properly test encryption/decryption/raw transfers
#
# For every scenario we need at least:
# - plain source dataset
# - encrypted source dataset
# - plain target path
# - encrypted target path
# - do a full transfer
# - do a incremental transfer
# Scenarios:
# - Raw transfer
# - Decryption transfer (--decrypt)
# - Encryption transfer (--encrypt)
# - Re-encryption transfer (--decrypt --encrypt)
class TestZfsEncryption(unittest2.TestCase): class TestZfsEncryption(unittest2.TestCase):
@ -65,8 +80,12 @@ test_target1/test_source2/fs2/sub encryption
self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget")
with patch('time.strftime', return_value="20101111000000"): with patch('time.strftime', return_value="20101111000000"):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty".split(" ")).run())
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot".split(" ")).run())
with patch('time.strftime', return_value="20101111000001"):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty".split(" ")).run())
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot".split(" ")).run())
r = shelltest("zfs get -r -t filesystem encryptionroot test_target1") r = shelltest("zfs get -r -t filesystem encryptionroot test_target1")
self.assertEqual(r, """ self.assertEqual(r, """
@ -96,21 +115,72 @@ test_target1/test_source2/fs2/sub encryptionroot -
self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget")
with patch('time.strftime', return_value="20101111000000"): with patch('time.strftime', return_value="20101111000000"):
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty".split(" ")).run())
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt".split(" ")).run()) self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot".split(" ")).run())
r = shelltest("zfs get encryption -H -o value test_target1/test_source1/fs1/encryptedsource test_target1/encryptedtarget/test_source1/fs1/encryptedsource") with patch('time.strftime', return_value="20101111000001"):
self.assertNotIn("off",r) self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty".split(" ")).run())
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot".split(" ")).run())
r = shelltest("zfs get -r -t filesystem encryptionroot test_target1")
self.assertEqual(r, """
NAME PROPERTY VALUE SOURCE
test_target1 encryptionroot - -
test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget -
test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget -
test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget -
test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource -
test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget -
test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget -
test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget -
test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget -
test_target1/test_source1 encryptionroot - -
test_target1/test_source1/fs1 encryptionroot - -
test_target1/test_source1/fs1/encryptedsource encryptionroot test_target1/test_source1/fs1/encryptedsource -
test_target1/test_source1/fs1/sub encryptionroot - -
test_target1/test_source2 encryptionroot - -
test_target1/test_source2/fs2 encryptionroot - -
test_target1/test_source2/fs2/sub encryptionroot - -
""")
def test_reencrypt(self): def test_reencrypt(self):
"""decrypt data and reencrypt on the otherside (--decrypt --encrypt) """ """reencrypt data (--decrypt --encrypt) """
# create encrypted target dataset self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource")
shelltest("echo 12345678 > /tmp/zfstest.key") self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget")
shelltest("zfs create -o keylocation=file:///tmp/zfstest.key -o keyformat=passphrase -o encryption=on test_target1/enc1")
with patch('time.strftime', return_value="20101111000000"): with patch('time.strftime', return_value="20101111000000"):
self.assertFalse(ZfsAutobackup("test test_target1/enc1 --allow-empty --verbose --no-progress".split(" ")).run()) self.assertFalse(ZfsAutobackup(
r = shelltest("zfs get encryption -H -o value test_target1/enc1/test_source1/fs1") "test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty".split(" ")).run())
self.assertNotIn("off",r) self.assertFalse(ZfsAutobackup(
"test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot".split(
" ")).run())
with patch('time.strftime', return_value="20101111000001"):
self.assertFalse(ZfsAutobackup(
"test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty".split(" ")).run())
self.assertFalse(ZfsAutobackup(
"test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot".split(
" ")).run())
r = shelltest("zfs get -r -t filesystem encryptionroot test_target1")
self.assertEqual(r, """
NAME PROPERTY VALUE SOURCE
test_target1 encryptionroot - -
test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget -
test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget -
test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget -
test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget -
test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget -
test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget -
test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget -
test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget -
test_target1/test_source1 encryptionroot - -
test_target1/test_source1/fs1 encryptionroot - -
test_target1/test_source1/fs1/encryptedsource encryptionroot - -
test_target1/test_source1/fs1/sub encryptionroot - -
test_target1/test_source2 encryptionroot - -
test_target1/test_source2/fs2 encryptionroot - -
test_target1/test_source2/fs2/sub encryptionroot - -
""")

2
tests/test_zfsautobackup.py
View File

@ -890,7 +890,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003
n=ZfsNode("test",l) n=ZfsNode("test",l)
d=ZfsDataset(n,"test_source1@test") d=ZfsDataset(n,"test_source1@test")
sp=d.send_pipe([], prev_snapshot=None, resume_token=None, show_progress=True, raw=False, output_pipes=[], send_properties=False) sp=d.send_pipe([], prev_snapshot=None, resume_token=None, show_progress=True, raw=False, output_pipes=[], send_properties=True, write_embedded=True)
with OutputIO() as buf: with OutputIO() as buf:

2
zfs_autobackup/ZfsAutobackup.py
View File

@ -268,7 +268,7 @@ class ZfsAutobackup:
also_other_snapshots=self.args.other_snapshots, also_other_snapshots=self.args.other_snapshots,
no_send=self.args.no_send, no_send=self.args.no_send,
destroy_incompatible=self.args.destroy_incompatible, destroy_incompatible=self.args.destroy_incompatible,
output_pipes=self.args.send_pipe, input_pipes=self.args.recv_pipe, decrypt=self.args.decrypt) output_pipes=self.args.send_pipe, input_pipes=self.args.recv_pipe, decrypt=self.args.decrypt, encrypt=self.args.encrypt)
except Exception as e: except Exception as e:
fail_count = fail_count + 1 fail_count = fail_count + 1
source_dataset.error("FAILED: " + str(e)) source_dataset.error("FAILED: " + str(e))

43
zfs_autobackup/ZfsDataset.py
View File

@ -494,7 +494,7 @@ class ZfsDataset:
return self.from_names(names[1:]) return self.from_names(names[1:])
def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, output_pipes): def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, write_embedded, output_pipes):
"""returns a pipe with zfs send output for this snapshot """returns a pipe with zfs send output for this snapshot
resume_token: resume sending from this token. (in that case we don't resume_token: resume sending from this token. (in that case we don't
@ -515,13 +515,13 @@ class ZfsDataset:
# all kind of performance options: # all kind of performance options:
if 'large_blocks' in features and "-L" in self.zfs_node.supported_send_options: if 'large_blocks' in features and "-L" in self.zfs_node.supported_send_options:
cmd.append("-L") # large block support (only if recordsize>128k which is seldomly used) cmd.append("--large-block") # large block support (only if recordsize>128k which is seldomly used)
if 'embedded_data' in features and "-e" in self.zfs_node.supported_send_options: if write_embedded and 'embedded_data' in features and "-e" in self.zfs_node.supported_send_options:
cmd.append("-e") # WRITE_EMBEDDED, more compact stream cmd.append("--embed") # WRITE_EMBEDDED, more compact stream
if "-c" in self.zfs_node.supported_send_options: if "-c" in self.zfs_node.supported_send_options:
cmd.append("-c") # use compressed WRITE records cmd.append("--compressed") # use compressed WRITE records
# raw? (send over encrypted data in its original encrypted form without decrypting) # raw? (send over encrypted data in its original encrypted form without decrypting)
if raw: if raw:
@ -529,8 +529,8 @@ class ZfsDataset:
# progress output # progress output
if show_progress: if show_progress:
cmd.append("-v") cmd.append("--verbose")
cmd.append("-P") cmd.append("--parsable")
# resume a previous send? (don't need more parameters in that case) # resume a previous send? (don't need more parameters in that case)
if resume_token: if resume_token:
@ -539,7 +539,7 @@ class ZfsDataset:
else: else:
# send properties # send properties
if send_properties: if send_properties:
cmd.append("-p") cmd.append("--props")
# incremental? # incremental?
if prev_snapshot: if prev_snapshot:
@ -632,7 +632,7 @@ class ZfsDataset:
def transfer_snapshot(self, target_snapshot, features, prev_snapshot, show_progress, def transfer_snapshot(self, target_snapshot, features, prev_snapshot, show_progress,
filter_properties, set_properties, ignore_recv_exit_code, resume_token, filter_properties, set_properties, ignore_recv_exit_code, resume_token,
raw, send_properties, output_pipes, input_pipes): raw, send_properties, write_embedded, output_pipes, input_pipes):
"""transfer this snapshot to target_snapshot. specify prev_snapshot for """transfer this snapshot to target_snapshot. specify prev_snapshot for
incremental transfer incremental transfer
@ -671,7 +671,7 @@ class ZfsDataset:
# do it # do it
pipe = self.send_pipe(features=features, show_progress=show_progress, prev_snapshot=prev_snapshot, pipe = self.send_pipe(features=features, show_progress=show_progress, prev_snapshot=prev_snapshot,
resume_token=resume_token, raw=raw, send_properties=send_properties, output_pipes=output_pipes) resume_token=resume_token, raw=raw, send_properties=send_properties, write_embedded=write_embedded, output_pipes=output_pipes)
target_snapshot.recv_pipe(pipe, features=features, filter_properties=filter_properties, target_snapshot.recv_pipe(pipe, features=features, filter_properties=filter_properties,
set_properties=set_properties, ignore_exit_code=ignore_recv_exit_code) set_properties=set_properties, ignore_exit_code=ignore_recv_exit_code)
@ -960,7 +960,7 @@ class ZfsDataset:
def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties, def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties,
ignore_recv_exit_code, holds, rollback, decrypt, also_other_snapshots, ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots,
no_send, destroy_incompatible, output_pipes, input_pipes): no_send, destroy_incompatible, output_pipes, input_pipes):
"""sync this dataset's snapshots to target_dataset, while also thinning """sync this dataset's snapshots to target_dataset, while also thinning
out old snapshots along the way. out old snapshots along the way.
@ -1007,8 +1007,12 @@ class ZfsDataset:
if rollback: if rollback:
target_dataset.rollback() target_dataset.rollback()
#defaults for these settings if there is no encryption stuff going on:
send_properties = True send_properties = True
raw = False raw = False
write_embedded = True
(active_filter_properties, active_set_properties) = self.get_allowed_properties(filter_properties, set_properties)
# source dataset encrypted? # source dataset encrypted?
if self.properties.get('encryption', 'off')!='off': if self.properties.get('encryption', 'off')!='off':
@ -1020,6 +1024,13 @@ class ZfsDataset:
# keep data encrypted by sending it raw (including properties) # keep data encrypted by sending it raw (including properties)
raw=True raw=True
# encrypt at target?
if encrypt and not raw:
# filter out encryption properties to let encryption on the target take place
active_filter_properties.extend(["keylocation","pbkdf2iters","keyformat", "encryption"])
write_embedded=False
# now actually transfer the snapshots # now actually transfer the snapshots
prev_source_snapshot = common_snapshot prev_source_snapshot = common_snapshot
source_snapshot = start_snapshot source_snapshot = start_snapshot
@ -1028,15 +1039,13 @@ class ZfsDataset:
# does target actually want it? # does target actually want it?
if target_snapshot not in target_obsoletes: if target_snapshot not in target_obsoletes:
# NOTE: should we let transfer_snapshot handle this?
(allowed_filter_properties, allowed_set_properties) = self.get_allowed_properties(filter_properties,
set_properties)
source_snapshot.transfer_snapshot(target_snapshot, features=features, source_snapshot.transfer_snapshot(target_snapshot, features=features,
prev_snapshot=prev_source_snapshot, show_progress=show_progress, prev_snapshot=prev_source_snapshot, show_progress=show_progress,
filter_properties=allowed_filter_properties, filter_properties=active_filter_properties,
set_properties=allowed_set_properties, set_properties=active_set_properties,
ignore_recv_exit_code=ignore_recv_exit_code, ignore_recv_exit_code=ignore_recv_exit_code,
resume_token=resume_token, raw=raw, send_properties=send_properties, output_pipes=output_pipes, input_pipes=input_pipes) resume_token=resume_token, write_embedded=write_embedded,raw=raw, send_properties=send_properties, output_pipes=output_pipes, input_pipes=input_pipes)
resume_token = None resume_token = None