greatly improved output of help (divided into sections)

zfs_autobackup/ZfsAuto.py

@@ -22,7 +22,7 @@ class ZfsAuto(object):
         self.args = self.parse_args(argv)
 
     def parse_args(self, argv):
-        """parse arguments, setup logging, check and adjust parameters"""
+        """parse common arguments, setup logging, check and adjust parameters"""
 
         parser=self.get_parser()
         args = parser.parse_args(argv)
@@ -83,85 +83,15 @@ class ZfsAuto(object):
 
         parser = argparse.ArgumentParser(description=self.HEADER,
                                          epilog='Full manual at: https://github.com/psy0rz/zfs_autobackup')
-        parser.add_argument('--ssh-config', metavar='CONFIG-FILE', default=None, help='Custom ssh client config')
-        parser.add_argument('--ssh-source', metavar='USER@HOST', default=None,
-                            help='Source host to get backup from.')
-        parser.add_argument('--ssh-target', metavar='USER@HOST', default=None,
-                            help='Target host to push backup to.')
-        parser.add_argument('--keep-source', metavar='SCHEDULE', type=str, default="10,1d1w,1w1m,1m1y",
-                            help='Thinning schedule for old source snapshots. Default: %(default)s')
-        parser.add_argument('--keep-target', metavar='SCHEDULE', type=str, default="10,1d1w,1w1m,1m1y",
-                            help='Thinning schedule for old target snapshots. Default: %(default)s')
 
+        #positional arguments
         parser.add_argument('backup_name', metavar='BACKUP-NAME', default=None, nargs='?',
-                            help='Name of the backup (you should set the zfs property "autobackup:backup-name" to '
+                            help='Name of the backup to select')
-                                 'true on filesystems you want to backup')
+
         parser.add_argument('target_path', metavar='TARGET-PATH', default=None, nargs='?',
-                            help='Target ZFS filesystem (optional: if not specified, zfs-autobackup will only operate '
+                            help='Target ZFS filesystem (optional)')
-                                 'as snapshot-tool on source)')
-
-        parser.add_argument('--pre-snapshot-cmd', metavar="COMMAND", default=[], action='append',
-                            help='Run COMMAND before snapshotting (can be used multiple times.')
-        parser.add_argument('--post-snapshot-cmd', metavar="COMMAND", default=[], action='append',
-                            help='Run COMMAND after snapshotting (can be used multiple times.')
-        parser.add_argument('--other-snapshots', action='store_true',
-                            help='Send over other snapshots as well, not just the ones created by this tool.')
-        parser.add_argument('--no-snapshot', action='store_true',
-                            help='Don\'t create new snapshots (useful for finishing uncompleted backups, or cleanups)')
-        parser.add_argument('--no-send', action='store_true',
-                            help='Don\'t send snapshots (useful for cleanups, or if you want a serperate send-cronjob)')
-        parser.add_argument('--no-thinning', action='store_true', help="Do not destroy any snapshots.")
-        parser.add_argument('--no-holds', action='store_true',
-                            help='Don\'t hold snapshots. (Faster. Allows you to destroy common snapshot.)')
-        parser.add_argument('--min-change', metavar='BYTES', type=int, default=1,
-                            help='Number of bytes written after which we consider a dataset changed (default %('
-                                 'default)s)')
-        parser.add_argument('--allow-empty', action='store_true',
-                            help='If nothing has changed, still create empty snapshots. (same as --min-change=0)')
-
-        parser.add_argument('--ignore-replicated', action='store_true', help=argparse.SUPPRESS)
-        parser.add_argument('--exclude-unchanged', action='store_true',
-                            help='Exclude datasets that have no changes since any last snapshot. (Useful in combination with proxmox HA replication)')
-        parser.add_argument('--exclude-received', action='store_true',
-                            help='Exclude datasets that have the origin of their autobackup: property as "received". '
-                                 'This can avoid recursive replication between two backup partners.')
-        parser.add_argument('--strip-path', metavar='N', default=0, type=int,
-                            help='Number of directories to strip from target path (use 1 when cloning zones between 2 '
-                                 'SmartOS machines)')
-
-        parser.add_argument('--clear-refreservation', action='store_true',
-                            help='Filter "refreservation" property. (recommended, safes space. same as '
-                                 '--filter-properties refreservation)')
-        parser.add_argument('--clear-mountpoint', action='store_true',
-                            help='Set property canmount=noauto for new datasets. (recommended, prevents mount '
-                                 'conflicts. same as --set-properties canmount=noauto)')
-        parser.add_argument('--filter-properties', metavar='PROPERTY,...', type=str,
-                            help='List of properties to "filter" when receiving filesystems. (you can still restore '
-                                 'them with zfs inherit -S)')
-        parser.add_argument('--set-properties', metavar='PROPERTY=VALUE,...', type=str,
-                            help='List of propererties to override when receiving filesystems. (you can still restore '
-                                 'them with zfs inherit -S)')
-        parser.add_argument('--rollback', action='store_true',
-                            help='Rollback changes to the latest target snapshot before starting. (normally you can '
-                                 'prevent changes by setting the readonly property on the target_path to on)')
-        parser.add_argument('--destroy-incompatible', action='store_true',
-                            help='Destroy incompatible snapshots on target. Use with care! (implies --rollback)')
-        parser.add_argument('--destroy-missing', metavar="SCHEDULE", type=str, default=None,
-                            help='Destroy datasets on target that are missing on the source. Specify the time since '
-                                 'the last snapshot, e.g: --destroy-missing 30d')
-        parser.add_argument('--ignore-transfer-errors', action='store_true',
-                            help='Ignore transfer errors (still checks if received filesystem exists. useful for '
-                                 'acltype errors)')
-
-        parser.add_argument('--decrypt', action='store_true',
-                            help='Decrypt data before sending it over.')
-
-        parser.add_argument('--encrypt', action='store_true',
-                            help='Encrypt data after receiving it.')
-
-        parser.add_argument('--zfs-compressed', action='store_true',
-                            help='Transfer blocks that already have zfs-compression as-is.')
 
+        # Basic options
         parser.add_argument('--test', '--dry-run', '-n', action='store_true',
                             help='Dry run, dont change anything, just show what would be done (still does all read-only '
                                  'operations)')
@@ -174,34 +104,20 @@ class ZfsAuto(object):
                             help='show zfs progress output. Enabled automaticly on ttys. (use --no-progress to disable)')
         parser.add_argument('--no-progress', action='store_true',
                             help=argparse.SUPPRESS)  # needed to workaround a zfs recv -v bug
-
-        parser.add_argument('--resume', action='store_true', help=argparse.SUPPRESS)
-        parser.add_argument('--raw', action='store_true', help=argparse.SUPPRESS)
-
-        # these things all do stuff by piping zfs send/recv IO
-        parser.add_argument('--send-pipe', metavar="COMMAND", default=[], action='append',
-                            help='pipe zfs send output through COMMAND (can be used multiple times)')
-        parser.add_argument('--recv-pipe', metavar="COMMAND", default=[], action='append',
-                            help='pipe zfs recv input through COMMAND (can be used multiple times)')
-        parser.add_argument('--compress', metavar='TYPE', default=None, nargs='?', const='zstd-fast',
-                            choices=compressors.choices(),
-                            help='Use compression during transfer, defaults to zstd-fast if TYPE is not specified. ({})'.format(
-                                ", ".join(compressors.choices())))
-        parser.add_argument('--rate', metavar='DATARATE', default=None,
-                            help='Limit data transfer rate (e.g. 128K. requires mbuffer.)')
-        parser.add_argument('--buffer', metavar='SIZE', default=None,
-                            help='Add zfs send and recv buffers to smooth out IO bursts. (e.g. 128M. requires mbuffer)')
-
-        parser.add_argument('--snapshot-format', metavar='FORMAT', default="{}-%Y%m%d%H%M%S",
-                            help='Snapshot naming format. Default: %(default)s')
-        parser.add_argument('--property-format', metavar='FORMAT', default="autobackup:{}",
-                            help='Select property naming format. Default: %(default)s')
-        parser.add_argument('--hold-format', metavar='FORMAT', default="zfs_autobackup:{}",
-                            help='Hold naming format. Default: %(default)s')
-
         parser.add_argument('--version', action='store_true',
                             help='Show version.')
 
+
+        # SSH options
+        group=parser.add_argument_group("SSH options")
+        group.add_argument('--ssh-config', metavar='CONFIG-FILE', default=None, help='Custom ssh client config')
+        group.add_argument('--ssh-source', metavar='USER@HOST', default=None,
+                            help='Source host to get backup from.')
+        group.add_argument('--ssh-target', metavar='USER@HOST', default=None,
+                            help='Target host to push backup to.')
+
+
+
         return (parser)
 
     def verbose(self, txt):

zfs_autobackup/ZfsAutobackup.py

@@ -1,5 +1,6 @@
 import time
 
+import argparse
 from .ZfsAuto import ZfsAuto
 
 from . import compressors
@@ -18,6 +19,115 @@ class ZfsAutobackup(ZfsAuto):
         super(ZfsAutobackup, self).__init__(argv, print_arguments)
 
 
+    def get_parser(self):
+        """extend common parser with  extra stuff needed for zfs-autobackup"""
+
+        parser=super(ZfsAutobackup, self).get_parser()
+
+
+        group=parser.add_argument_group("Selection options")
+        group.add_argument('--ignore-replicated', action='store_true', help=argparse.SUPPRESS)
+        group.add_argument('--exclude-unchanged', action='store_true',
+                            help='Exclude datasets that have no changes since any last snapshot. (Useful in combination with proxmox HA replication)')
+        group.add_argument('--exclude-received', action='store_true',
+                            help='Exclude datasets that have the origin of their autobackup: property as "received". '
+                                 'This can avoid recursive replication between two backup partners.')
+        group.add_argument('--property-format', metavar='FORMAT', default="autobackup:{}",
+                            help='Dataset selection string format. Default: %(default)s')
+
+        group=parser.add_argument_group("Snapshot options")
+        group.add_argument('--no-snapshot', action='store_true',
+                            help='Don\'t create new snapshots (useful for finishing uncompleted backups, or cleanups)')
+        group.add_argument('--pre-snapshot-cmd', metavar="COMMAND", default=[], action='append',
+                            help='Run COMMAND before snapshotting (can be used multiple times.')
+        group.add_argument('--post-snapshot-cmd', metavar="COMMAND", default=[], action='append',
+                            help='Run COMMAND after snapshotting (can be used multiple times.')
+        group.add_argument('--other-snapshots', action='store_true',
+                            help='Send over other snapshots as well, not just the ones created by this tool.')
+        group.add_argument('--min-change', metavar='BYTES', type=int, default=1,
+                            help='Number of bytes written after which we consider a dataset changed (default %('
+                                 'default)s)')
+        group.add_argument('--allow-empty', action='store_true',
+                            help='If nothing has changed, still create empty snapshots. (Faster. Same as --min-change=0)')
+        group.add_argument('--snapshot-format', metavar='FORMAT', default="{}-%Y%m%d%H%M%S",
+                            help='ZFS Snapshot string format. Default: %(default)s')
+
+        group=parser.add_argument_group("Transfer options")
+        group.add_argument('--no-send', action='store_true',
+                            help='Don\'t transfer snapshots (useful for cleanups, or if you want a serperate send-cronjob)')
+        group.add_argument('--no-holds', action='store_true',
+                            help='Don\'t hold snapshots. (Faster. Allows you to destroy common snapshot.)')
+        group.add_argument('--strip-path', metavar='N', default=0, type=int,
+                            help='Number of directories to strip from target path (use 1 when cloning zones between 2 '
+                                 'SmartOS machines)')
+        group.add_argument('--clear-refreservation', action='store_true',
+                            help='Filter "refreservation" property. (recommended, safes space. same as '
+                                 '--filter-properties refreservation)')
+        group.add_argument('--clear-mountpoint', action='store_true',
+                            help='Set property canmount=noauto for new datasets. (recommended, prevents mount '
+                                 'conflicts. same as --set-properties canmount=noauto)')
+        group.add_argument('--filter-properties', metavar='PROPERTY,...', type=str,
+                            help='List of properties to "filter" when receiving filesystems. (you can still restore '
+                                 'them with zfs inherit -S)')
+        group.add_argument('--set-properties', metavar='PROPERTY=VALUE,...', type=str,
+                            help='List of propererties to override when receiving filesystems. (you can still restore '
+                                 'them with zfs inherit -S)')
+        group.add_argument('--rollback', action='store_true',
+                            help='Rollback changes to the latest target snapshot before starting. (normally you can '
+                                 'prevent changes by setting the readonly property on the target_path to on)')
+        group.add_argument('--destroy-incompatible', action='store_true',
+                            help='Destroy incompatible snapshots on target. Use with care! (implies --rollback)')
+        group.add_argument('--ignore-transfer-errors', action='store_true',
+                            help='Ignore transfer errors (still checks if received filesystem exists. useful for '
+                                 'acltype errors)')
+
+        group.add_argument('--decrypt', action='store_true',
+                            help='Decrypt data before sending it over.')
+        group.add_argument('--encrypt', action='store_true',
+                            help='Encrypt data after receiving it.')
+
+        group.add_argument('--zfs-compressed', action='store_true',
+                            help='Transfer blocks that already have zfs-compression as-is.')
+        group.add_argument('--hold-format', metavar='FORMAT', default="zfs_autobackup:{}",
+                            help='ZFS hold string format. Default: %(default)s')
+
+
+
+        group=parser.add_argument_group("ZFS send/recv pipes")
+        group.add_argument('--compress', metavar='TYPE', default=None, nargs='?', const='zstd-fast',
+                            choices=compressors.choices(),
+                            help='Use compression during transfer, defaults to zstd-fast if TYPE is not specified. ({})'.format(
+                                ", ".join(compressors.choices())))
+        group.add_argument('--rate', metavar='DATARATE', default=None,
+                            help='Limit data transfer rate (e.g. 128K. requires mbuffer.)')
+        group.add_argument('--buffer', metavar='SIZE', default=None,
+                            help='Add zfs send and recv buffers to smooth out IO bursts. (e.g. 128M. requires mbuffer)')
+        group.add_argument('--send-pipe', metavar="COMMAND", default=[], action='append',
+                            help='pipe zfs send output through COMMAND (can be used multiple times)')
+        group.add_argument('--recv-pipe', metavar="COMMAND", default=[], action='append',
+                            help='pipe zfs recv input through COMMAND (can be used multiple times)')
+
+
+        group=parser.add_argument_group("Thinner options")
+        group.add_argument('--no-thinning', action='store_true', help="Do not destroy any snapshots.")
+        group.add_argument('--keep-source', metavar='SCHEDULE', type=str, default="10,1d1w,1w1m,1m1y",
+                            help='Thinning schedule for old source snapshots. Default: %(default)s')
+        group.add_argument('--keep-target', metavar='SCHEDULE', type=str, default="10,1d1w,1w1m,1m1y",
+                            help='Thinning schedule for old target snapshots. Default: %(default)s')
+        group.add_argument('--destroy-missing', metavar="SCHEDULE", type=str, default=None,
+                            help='Destroy datasets on target that are missing on the source. Specify the time since '
+                                 'the last snapshot, e.g: --destroy-missing 30d')
+
+
+        #obsolete
+        parser.add_argument('--resume', action='store_true', help=argparse.SUPPRESS)
+        parser.add_argument('--raw', action='store_true', help=argparse.SUPPRESS)
+
+
+
+
+        return (parser)
+
     def progress(self, txt):
         self.log.progress(txt)
 

zfs_autobackup/ZfsNode.py

@@ -55,6 +55,7 @@ class ZfsNode(ExecuteNode):
         ExecuteNode.__init__(self, ssh_config=ssh_config, ssh_to=ssh_to, readonly=readonly, debug_output=debug_output)
 
     def thin(self, objects, keep_objects):
+        # NOTE: if thinning is disabled with --no-thinning, self.__thinner will be none.
         if self.__thinner is not None:
             return self.__thinner.thin(objects, keep_objects)
         else: