added compression, fixes #40

zfs_autobackup/ZfsAutobackup.py

@@ -2,6 +2,7 @@ import argparse
 import sys
 import time
 
+from zfs_autobackup import compressors
 from zfs_autobackup.ExecuteNode import ExecuteNode
 from zfs_autobackup.Thinner import Thinner
 from zfs_autobackup.ZfsDataset import ZfsDataset
@@ -11,6 +12,7 @@ from zfs_autobackup.ThinnerRule import ThinnerRule
 
 
 
+
 class ZfsAutobackup:
     """main class"""
 
@@ -121,6 +123,9 @@ class ZfsAutobackup:
         parser.add_argument('--exclude-received', action='store_true',
                             help=argparse.SUPPRESS)  # probably never needed anymore
 
+        parser.add_argument('--compress', metavar='TYPE', default=None, choices=compressors.choices(), help='Use compression during transfer ({})'.format(", ".join(compressors.choices())))
+
+
         # note args is the only global variable we use, since its a global readonly setting anyway
         args = parser.parse_args(argv)
 
@@ -261,28 +266,44 @@ class ZfsAutobackup:
         if self.args.progress:
             self.clear_progress()
 
-    def get_recv_pipes(self):
+    def get_send_pipes(self, logger):
+        """determine the zfs send pipe"""
 
         ret=[]
 
-        for recv_pipe in self.args.recv_pipe:
-            ret.extend(recv_pipe.split(" "))
+        # compression
+        if self.args.compress!=None:
             ret.append(ExecuteNode.PIPE)
-            self.verbose("Added recv pipe: {}".format(recv_pipe))
-
-        return ret
-
-    def get_send_pipes(self):
-
-        ret=[]
+            cmd=compressors.compress_cmd(self.args.compress)
+            ret.extend(cmd)
+            logger("zfs send compression   : {}".format(" ".join(cmd)))
 
+        # custom pipes
         for send_pipe in self.args.send_pipe:
             ret.append(ExecuteNode.PIPE)
             ret.extend(send_pipe.split(" "))
-            self.verbose("Added send pipe: {}".format(send_pipe))
+            logger("zfs send custom pipe   : {}".format(send_pipe))
 
         return ret
 
+    def get_recv_pipes(self, logger):
+
+        ret=[]
+
+        # custom pipes
+        for recv_pipe in self.args.recv_pipe:
+            ret.extend(recv_pipe.split(" "))
+            ret.append(ExecuteNode.PIPE)
+            logger("zfs recv custom pipe   : {}".format(recv_pipe))
+
+        # decompression
+        if self.args.compress!=None:
+            cmd=compressors.decompress_cmd(self.args.compress)
+            ret.extend(cmd)
+            ret.append(ExecuteNode.PIPE)
+            logger("zfs recv decompression : {}".format(" ".join(cmd)))
+
+        return ret
 
     # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters:
     def sync_datasets(self, source_node, source_datasets, target_node):
@@ -292,8 +313,8 @@ class ZfsAutobackup:
         :type source_node: ZfsNode
         """
 
-        send_pipes=self.get_send_pipes()
-        recv_pipes=self.get_recv_pipes()
+        send_pipes=self.get_send_pipes(source_node.verbose)
+        recv_pipes=self.get_recv_pipes(target_node.verbose)
 
         fail_count = 0
         count = 0

zfs_autobackup/compressors.py

@@ -0,0 +1,69 @@
+# Adopted from Syncoid :)
+
+# this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved
+# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17.  A copy should also be available in this
+# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
+
+COMPRESS_CMDS = {
+    'gzip': {
+        'cmd': 'gzip',
+        'args': [ '-3' ],
+        'dcmd': 'zcat',
+        'dargs': [],
+    },
+    'pigz-fast': {
+        'cmd': 'pigz',
+        'args': [ '-3' ],
+        'dcmd': 'pigz',
+        'dargs': [ '-dc' ],
+    },
+    'pigz-slow': {
+        'cmd': 'pigz',
+        'args': [ '-9' ],
+        'dcmd': 'pigz',
+        'dargs': [ '-dc' ],
+    },
+    'zstd-fast': {
+        'cmd': 'zstdmt',
+        'args': [ '-3' ],
+        'dcmd': 'zstdmt',
+        'dargs': [ '-dc' ],
+    },
+    'zstd-slow': {
+        'cmd': 'zstdmt',
+        'args': [ '-19' ],
+        'dcmd': 'zstdmt',
+        'dargs': [ '-dc' ],
+    },
+    'xz': {
+        'cmd': 'xz',
+        'args': [],
+        'dcmd': 'xz',
+        'dargs': [ '-d' ],
+    },
+    'lzo': {
+        'cmd': 'lzop',
+        'args': [],
+        'dcmd': 'lzop',
+        'dargs': [ '-dfc' ],
+    },
+    'lz4': {
+        'cmd': 'lz4',
+        'args': [],
+        'dcmd': 'lz4',
+        'dargs': [ '-dc' ],
+    },
+}
+
+def compress_cmd(compressor):
+    ret=[ COMPRESS_CMDS[compressor]['cmd'] ]
+    ret.extend( COMPRESS_CMDS[compressor]['args'])
+    return ret
+
+def decompress_cmd(compressor):
+    ret= [ COMPRESS_CMDS[compressor]['dcmd'] ]
+    ret.extend(COMPRESS_CMDS[compressor]['dargs'])
+    return ret
+
+def choices():
+    return COMPRESS_CMDS.keys()