diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 171c125..9217677 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -2,6 +2,7 @@ import argparse import sys import time +from zfs_autobackup import compressors from zfs_autobackup.ExecuteNode import ExecuteNode from zfs_autobackup.Thinner import Thinner from zfs_autobackup.ZfsDataset import ZfsDataset @@ -11,6 +12,7 @@ from zfs_autobackup.ThinnerRule import ThinnerRule + class ZfsAutobackup: """main class""" @@ -121,6 +123,9 @@ class ZfsAutobackup: parser.add_argument('--exclude-received', action='store_true', help=argparse.SUPPRESS) # probably never needed anymore + parser.add_argument('--compress', metavar='TYPE', default=None, choices=compressors.choices(), help='Use compression during transfer ({})'.format(", ".join(compressors.choices()))) + + # note args is the only global variable we use, since its a global readonly setting anyway args = parser.parse_args(argv) @@ -261,28 +266,44 @@ class ZfsAutobackup: if self.args.progress: self.clear_progress() - def get_recv_pipes(self): + def get_send_pipes(self, logger): + """determine the zfs send pipe""" ret=[] - for recv_pipe in self.args.recv_pipe: - ret.extend(recv_pipe.split(" ")) + # compression + if self.args.compress!=None: ret.append(ExecuteNode.PIPE) - self.verbose("Added recv pipe: {}".format(recv_pipe)) - - return ret - - def get_send_pipes(self): - - ret=[] + cmd=compressors.compress_cmd(self.args.compress) + ret.extend(cmd) + logger("zfs send compression : {}".format(" ".join(cmd))) + # custom pipes for send_pipe in self.args.send_pipe: ret.append(ExecuteNode.PIPE) ret.extend(send_pipe.split(" ")) - self.verbose("Added send pipe: {}".format(send_pipe)) + logger("zfs send custom pipe : {}".format(send_pipe)) return ret + def get_recv_pipes(self, logger): + + ret=[] + + # custom pipes + for recv_pipe in self.args.recv_pipe: + ret.extend(recv_pipe.split(" ")) + ret.append(ExecuteNode.PIPE) + logger("zfs recv custom pipe : {}".format(recv_pipe)) + + # decompression + if self.args.compress!=None: + cmd=compressors.decompress_cmd(self.args.compress) + ret.extend(cmd) + ret.append(ExecuteNode.PIPE) + logger("zfs recv decompression : {}".format(" ".join(cmd))) + + return ret # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: def sync_datasets(self, source_node, source_datasets, target_node): @@ -292,8 +313,8 @@ class ZfsAutobackup: :type source_node: ZfsNode """ - send_pipes=self.get_send_pipes() - recv_pipes=self.get_recv_pipes() + send_pipes=self.get_send_pipes(source_node.verbose) + recv_pipes=self.get_recv_pipes(target_node.verbose) fail_count = 0 count = 0 diff --git a/zfs_autobackup/compressors.py b/zfs_autobackup/compressors.py new file mode 100644 index 0000000..efff265 --- /dev/null +++ b/zfs_autobackup/compressors.py @@ -0,0 +1,69 @@ +# Adopted from Syncoid :) + +# this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved +# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this +# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. + +COMPRESS_CMDS = { + 'gzip': { + 'cmd': 'gzip', + 'args': [ '-3' ], + 'dcmd': 'zcat', + 'dargs': [], + }, + 'pigz-fast': { + 'cmd': 'pigz', + 'args': [ '-3' ], + 'dcmd': 'pigz', + 'dargs': [ '-dc' ], + }, + 'pigz-slow': { + 'cmd': 'pigz', + 'args': [ '-9' ], + 'dcmd': 'pigz', + 'dargs': [ '-dc' ], + }, + 'zstd-fast': { + 'cmd': 'zstdmt', + 'args': [ '-3' ], + 'dcmd': 'zstdmt', + 'dargs': [ '-dc' ], + }, + 'zstd-slow': { + 'cmd': 'zstdmt', + 'args': [ '-19' ], + 'dcmd': 'zstdmt', + 'dargs': [ '-dc' ], + }, + 'xz': { + 'cmd': 'xz', + 'args': [], + 'dcmd': 'xz', + 'dargs': [ '-d' ], + }, + 'lzo': { + 'cmd': 'lzop', + 'args': [], + 'dcmd': 'lzop', + 'dargs': [ '-dfc' ], + }, + 'lz4': { + 'cmd': 'lz4', + 'args': [], + 'dcmd': 'lz4', + 'dargs': [ '-dc' ], + }, +} + +def compress_cmd(compressor): + ret=[ COMPRESS_CMDS[compressor]['cmd'] ] + ret.extend( COMPRESS_CMDS[compressor]['args']) + return ret + +def decompress_cmd(compressor): + ret= [ COMPRESS_CMDS[compressor]['dcmd'] ] + ret.extend(COMPRESS_CMDS[compressor]['dargs']) + return ret + +def choices(): + return COMPRESS_CMDS.keys()