mirror of
https://github.com/psy0rz/zfs_autobackup.git
synced 2025-04-15 22:50:32 +03:00
proper encryption/decryption support. also fixes #60
This commit is contained in:
Edwin Eefting
parent
7696d8c16d
commit
176f04b302
14
README.md
14
README.md
@ -375,16 +375,24 @@ Snapshots on the source that still have to be send to the target wont be destroy
|
||||
In normal operation datasets are transferred unaltered:
|
||||
|
||||
* Source datasets that are encrypted will be send over as such and stay encrypted at the target side. (In ZFS this is called raw-mode) You dont need keys at the target side if you dont want to access the data.
|
||||
* Source datasets that are plain will stay that way on the target. Even if the specified target-path IS encrypted.
|
||||
* Source datasets that are plain will stay that way on the target. (Even if the specified target-path IS encrypted.)
|
||||
|
||||
Basically you dont have to do anything or worry about anything.
|
||||
|
||||
### Decrypting/encrypting
|
||||
|
||||
If you want to alter the encryption-state of a dataset you have several options:
|
||||
Things get different if you want to change the encryption-state of a dataset during transfer:
|
||||
|
||||
* If you want to decrypt encrypted datasets before sending them, you should use the `--decrypt` option. Datasets will then be stored plain at the target.
|
||||
* If you want to encrypt plain datasets when they are received, you should use the `--encrypt` option. Datasets will then be stored encrypted at the target. (Datasets that are already encrypted will still be sent over unaltered!) You are responsible for creating the target-path with encryption enabled.
|
||||
* If you want to encrypt plain datasets when they are received, you should use the `--encrypt` option. Datasets will then be stored encrypted at the target. (Datasets that are already encrypted will still be sent over unaltered!)
|
||||
* If you also want re-encrypt encrypted datasets with the target-side encryption you can use both options.
|
||||
|
||||
Note 1: The --encrypt option will rely on inheriting encryption parameters from the parent datasets on the parent side. You are responsible for setting those up and loading the keys. So --encrypt is no guarantee for encryption, if its not setup, it cant be encrypted.
|
||||
|
||||
Note 2: Decide what you want at an early stage: If you change the --encrypt or --decrypt parameter at a later time you might get weird and wonderfull errors. (nothing dangerous)
|
||||
|
||||
I'll add some tips when the issues start to get in on github. :)
|
||||
|
||||
## Tips
|
||||
|
||||
* Use ```--debug``` if something goes wrong and you want to see the commands that are executed. This will also stop at the first error.
|
||||
|
||||
@ -2,6 +2,21 @@ from zfs_autobackup.CmdPipe import CmdPipe
|
||||
from basetest import *
|
||||
import time
|
||||
|
||||
# We have to do a LOT to properly test encryption/decryption/raw transfers
|
||||
#
|
||||
# For every scenario we need at least:
|
||||
# - plain source dataset
|
||||
# - encrypted source dataset
|
||||
# - plain target path
|
||||
# - encrypted target path
|
||||
# - do a full transfer
|
||||
# - do a incremental transfer
|
||||
|
||||
# Scenarios:
|
||||
# - Raw transfer
|
||||
# - Decryption transfer (--decrypt)
|
||||
# - Encryption transfer (--encrypt)
|
||||
# - Re-encryption transfer (--decrypt --encrypt)
|
||||
|
||||
class TestZfsEncryption(unittest2.TestCase):
|
||||
|
||||
@ -65,8 +80,12 @@ test_target1/test_source2/fs2/sub encryption
|
||||
self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget")
|
||||
|
||||
with patch('time.strftime', return_value="20101111000000"):
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt".split(" ")).run())
|
||||
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt".split(" ")).run())
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty".split(" ")).run())
|
||||
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot".split(" ")).run())
|
||||
|
||||
with patch('time.strftime', return_value="20101111000001"):
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --decrypt --allow-empty".split(" ")).run())
|
||||
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --decrypt --no-snapshot".split(" ")).run())
|
||||
|
||||
r = shelltest("zfs get -r -t filesystem encryptionroot test_target1")
|
||||
self.assertEqual(r, """
|
||||
@ -96,21 +115,72 @@ test_target1/test_source2/fs2/sub encryptionroot -
|
||||
self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget")
|
||||
|
||||
with patch('time.strftime', return_value="20101111000000"):
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt".split(" ")).run())
|
||||
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt".split(" ")).run())
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty".split(" ")).run())
|
||||
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot".split(" ")).run())
|
||||
|
||||
r = shelltest("zfs get encryption -H -o value test_target1/test_source1/fs1/encryptedsource test_target1/encryptedtarget/test_source1/fs1/encryptedsource")
|
||||
self.assertNotIn("off",r)
|
||||
with patch('time.strftime', return_value="20101111000001"):
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-progress --encrypt --debug --allow-empty".split(" ")).run())
|
||||
self.assertFalse(ZfsAutobackup("test test_target1/encryptedtarget --verbose --no-progress --encrypt --debug --no-snapshot".split(" ")).run())
|
||||
|
||||
r = shelltest("zfs get -r -t filesystem encryptionroot test_target1")
|
||||
self.assertEqual(r, """
|
||||
NAME PROPERTY VALUE SOURCE
|
||||
test_target1 encryptionroot - -
|
||||
test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget/test_source1/fs1/encryptedsource -
|
||||
test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/test_source1 encryptionroot - -
|
||||
test_target1/test_source1/fs1 encryptionroot - -
|
||||
test_target1/test_source1/fs1/encryptedsource encryptionroot test_target1/test_source1/fs1/encryptedsource -
|
||||
test_target1/test_source1/fs1/sub encryptionroot - -
|
||||
test_target1/test_source2 encryptionroot - -
|
||||
test_target1/test_source2/fs2 encryptionroot - -
|
||||
test_target1/test_source2/fs2/sub encryptionroot - -
|
||||
""")
|
||||
|
||||
def test_reencrypt(self):
|
||||
"""decrypt data and reencrypt on the otherside (--decrypt --encrypt) """
|
||||
def test_reencrypt(self):
|
||||
"""reencrypt data (--decrypt --encrypt) """
|
||||
|
||||
# create encrypted target dataset
|
||||
shelltest("echo 12345678 > /tmp/zfstest.key")
|
||||
shelltest("zfs create -o keylocation=file:///tmp/zfstest.key -o keyformat=passphrase -o encryption=on test_target1/enc1")
|
||||
self.prepare_encrypted_dataset("11111111", "test_source1/fs1/encryptedsource")
|
||||
self.prepare_encrypted_dataset("22222222", "test_target1/encryptedtarget")
|
||||
|
||||
with patch('time.strftime', return_value="20101111000000"):
|
||||
self.assertFalse(ZfsAutobackup("test test_target1/enc1 --allow-empty --verbose --no-progress".split(" ")).run())
|
||||
r = shelltest("zfs get encryption -H -o value test_target1/enc1/test_source1/fs1")
|
||||
self.assertNotIn("off",r)
|
||||
self.assertFalse(ZfsAutobackup(
|
||||
"test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty".split(" ")).run())
|
||||
self.assertFalse(ZfsAutobackup(
|
||||
"test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot".split(
|
||||
" ")).run())
|
||||
|
||||
with patch('time.strftime', return_value="20101111000001"):
|
||||
self.assertFalse(ZfsAutobackup(
|
||||
"test test_target1 --verbose --no-progress --decrypt --encrypt --debug --allow-empty".split(" ")).run())
|
||||
self.assertFalse(ZfsAutobackup(
|
||||
"test test_target1/encryptedtarget --verbose --no-progress --decrypt --encrypt --debug --no-snapshot".split(
|
||||
" ")).run())
|
||||
|
||||
r = shelltest("zfs get -r -t filesystem encryptionroot test_target1")
|
||||
self.assertEqual(r, """
|
||||
NAME PROPERTY VALUE SOURCE
|
||||
test_target1 encryptionroot - -
|
||||
test_target1/encryptedtarget encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/encryptedtarget/test_source1 encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/encryptedtarget/test_source1/fs1 encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/encryptedtarget/test_source1/fs1/encryptedsource encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/encryptedtarget/test_source1/fs1/sub encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/encryptedtarget/test_source2 encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/encryptedtarget/test_source2/fs2 encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/encryptedtarget/test_source2/fs2/sub encryptionroot test_target1/encryptedtarget -
|
||||
test_target1/test_source1 encryptionroot - -
|
||||
test_target1/test_source1/fs1 encryptionroot - -
|
||||
test_target1/test_source1/fs1/encryptedsource encryptionroot - -
|
||||
test_target1/test_source1/fs1/sub encryptionroot - -
|
||||
test_target1/test_source2 encryptionroot - -
|
||||
test_target1/test_source2/fs2 encryptionroot - -
|
||||
test_target1/test_source2/fs2/sub encryptionroot - -
|
||||
""")
|
||||
|
||||
|
||||
@ -890,7 +890,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003
|
||||
n=ZfsNode("test",l)
|
||||
d=ZfsDataset(n,"test_source1@test")
|
||||
|
||||
sp=d.send_pipe([], prev_snapshot=None, resume_token=None, show_progress=True, raw=False, output_pipes=[], send_properties=False)
|
||||
sp=d.send_pipe([], prev_snapshot=None, resume_token=None, show_progress=True, raw=False, output_pipes=[], send_properties=True, write_embedded=True)
|
||||
|
||||
|
||||
with OutputIO() as buf:
|
||||
|
||||
@ -268,7 +268,7 @@ class ZfsAutobackup:
|
||||
also_other_snapshots=self.args.other_snapshots,
|
||||
no_send=self.args.no_send,
|
||||
destroy_incompatible=self.args.destroy_incompatible,
|
||||
output_pipes=self.args.send_pipe, input_pipes=self.args.recv_pipe, decrypt=self.args.decrypt)
|
||||
output_pipes=self.args.send_pipe, input_pipes=self.args.recv_pipe, decrypt=self.args.decrypt, encrypt=self.args.encrypt)
|
||||
except Exception as e:
|
||||
fail_count = fail_count + 1
|
||||
source_dataset.error("FAILED: " + str(e))
|
||||
|
||||
@ -494,7 +494,7 @@ class ZfsDataset:
|
||||
|
||||
return self.from_names(names[1:])
|
||||
|
||||
def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, output_pipes):
|
||||
def send_pipe(self, features, prev_snapshot, resume_token, show_progress, raw, send_properties, write_embedded, output_pipes):
|
||||
"""returns a pipe with zfs send output for this snapshot
|
||||
|
||||
resume_token: resume sending from this token. (in that case we don't
|
||||
@ -515,13 +515,13 @@ class ZfsDataset:
|
||||
|
||||
# all kind of performance options:
|
||||
if 'large_blocks' in features and "-L" in self.zfs_node.supported_send_options:
|
||||
cmd.append("-L") # large block support (only if recordsize>128k which is seldomly used)
|
||||
cmd.append("--large-block") # large block support (only if recordsize>128k which is seldomly used)
|
||||
|
||||
if 'embedded_data' in features and "-e" in self.zfs_node.supported_send_options:
|
||||
cmd.append("-e") # WRITE_EMBEDDED, more compact stream
|
||||
if write_embedded and 'embedded_data' in features and "-e" in self.zfs_node.supported_send_options:
|
||||
cmd.append("--embed") # WRITE_EMBEDDED, more compact stream
|
||||
|
||||
if "-c" in self.zfs_node.supported_send_options:
|
||||
cmd.append("-c") # use compressed WRITE records
|
||||
cmd.append("--compressed") # use compressed WRITE records
|
||||
|
||||
# raw? (send over encrypted data in its original encrypted form without decrypting)
|
||||
if raw:
|
||||
@ -529,8 +529,8 @@ class ZfsDataset:
|
||||
|
||||
# progress output
|
||||
if show_progress:
|
||||
cmd.append("-v")
|
||||
cmd.append("-P")
|
||||
cmd.append("--verbose")
|
||||
cmd.append("--parsable")
|
||||
|
||||
# resume a previous send? (don't need more parameters in that case)
|
||||
if resume_token:
|
||||
@ -539,7 +539,7 @@ class ZfsDataset:
|
||||
else:
|
||||
# send properties
|
||||
if send_properties:
|
||||
cmd.append("-p")
|
||||
cmd.append("--props")
|
||||
|
||||
# incremental?
|
||||
if prev_snapshot:
|
||||
@ -632,7 +632,7 @@ class ZfsDataset:
|
||||
|
||||
def transfer_snapshot(self, target_snapshot, features, prev_snapshot, show_progress,
|
||||
filter_properties, set_properties, ignore_recv_exit_code, resume_token,
|
||||
raw, send_properties, output_pipes, input_pipes):
|
||||
raw, send_properties, write_embedded, output_pipes, input_pipes):
|
||||
"""transfer this snapshot to target_snapshot. specify prev_snapshot for
|
||||
incremental transfer
|
||||
|
||||
@ -671,7 +671,7 @@ class ZfsDataset:
|
||||
|
||||
# do it
|
||||
pipe = self.send_pipe(features=features, show_progress=show_progress, prev_snapshot=prev_snapshot,
|
||||
resume_token=resume_token, raw=raw, send_properties=send_properties, output_pipes=output_pipes)
|
||||
resume_token=resume_token, raw=raw, send_properties=send_properties, write_embedded=write_embedded, output_pipes=output_pipes)
|
||||
target_snapshot.recv_pipe(pipe, features=features, filter_properties=filter_properties,
|
||||
set_properties=set_properties, ignore_exit_code=ignore_recv_exit_code)
|
||||
|
||||
@ -960,7 +960,7 @@ class ZfsDataset:
|
||||
|
||||
|
||||
def sync_snapshots(self, target_dataset, features, show_progress, filter_properties, set_properties,
|
||||
ignore_recv_exit_code, holds, rollback, decrypt, also_other_snapshots,
|
||||
ignore_recv_exit_code, holds, rollback, decrypt, encrypt, also_other_snapshots,
|
||||
no_send, destroy_incompatible, output_pipes, input_pipes):
|
||||
"""sync this dataset's snapshots to target_dataset, while also thinning
|
||||
out old snapshots along the way.
|
||||
@ -1007,8 +1007,12 @@ class ZfsDataset:
|
||||
if rollback:
|
||||
target_dataset.rollback()
|
||||
|
||||
#defaults for these settings if there is no encryption stuff going on:
|
||||
send_properties = True
|
||||
raw = False
|
||||
write_embedded = True
|
||||
|
||||
(active_filter_properties, active_set_properties) = self.get_allowed_properties(filter_properties, set_properties)
|
||||
|
||||
# source dataset encrypted?
|
||||
if self.properties.get('encryption', 'off')!='off':
|
||||
@ -1020,6 +1024,13 @@ class ZfsDataset:
|
||||
# keep data encrypted by sending it raw (including properties)
|
||||
raw=True
|
||||
|
||||
# encrypt at target?
|
||||
if encrypt and not raw:
|
||||
# filter out encryption properties to let encryption on the target take place
|
||||
active_filter_properties.extend(["keylocation","pbkdf2iters","keyformat", "encryption"])
|
||||
write_embedded=False
|
||||
|
||||
|
||||
# now actually transfer the snapshots
|
||||
prev_source_snapshot = common_snapshot
|
||||
source_snapshot = start_snapshot
|
||||
@ -1028,15 +1039,13 @@ class ZfsDataset:
|
||||
|
||||
# does target actually want it?
|
||||
if target_snapshot not in target_obsoletes:
|
||||
# NOTE: should we let transfer_snapshot handle this?
|
||||
(allowed_filter_properties, allowed_set_properties) = self.get_allowed_properties(filter_properties,
|
||||
set_properties)
|
||||
|
||||
source_snapshot.transfer_snapshot(target_snapshot, features=features,
|
||||
prev_snapshot=prev_source_snapshot, show_progress=show_progress,
|
||||
filter_properties=allowed_filter_properties,
|
||||
set_properties=allowed_set_properties,
|
||||
filter_properties=active_filter_properties,
|
||||
set_properties=active_set_properties,
|
||||
ignore_recv_exit_code=ignore_recv_exit_code,
|
||||
resume_token=resume_token, raw=raw, send_properties=send_properties, output_pipes=output_pipes, input_pipes=input_pipes)
|
||||
resume_token=resume_token, write_embedded=write_embedded,raw=raw, send_properties=send_properties, output_pipes=output_pipes, input_pipes=input_pipes)
|
||||
|
||||
resume_token = None
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user