third-party-mirrors/zfs_autobackup
1
0
Fork 1
mirror of https://github.com/psy0rz/zfs_autobackup.git synced 2025-06-11 02:02:05 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

reformatting. changed version number

Browse Source
This commit is contained in:
Your Name 2024-09-24 18:22:17 +02:00
parent 5f2e686a1b
commit 07542365ac
18 changed files with 268 additions and 276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
zfs_autobackup/BlockHasher.py
View File

@ -19,15 +19,14 @@ class BlockHasher():
def __init__(self, count=10000, bs=4096, hash_class=hashlib.sha1, skip=0): def __init__(self, count=10000, bs=4096, hash_class=hashlib.sha1, skip=0):
self.count = count self.count = count
self.bs = bs self.bs = bs
self.chunk_size=bs*count self.chunk_size = bs * count
self.hash_class = hash_class self.hash_class = hash_class
# self.coverage=coverage # self.coverage=coverage
self.skip=skip self.skip = skip
self._skip_count=0 self._skip_count = 0
self.stats_total_bytes=0
self.stats_total_bytes = 0
def _seek_next_chunk(self, fh, fsize): def _seek_next_chunk(self, fh, fsize):
"""seek fh to next chunk and update skip counter. """seek fh to next chunk and update skip counter.
@ -37,8 +36,8 @@ class BlockHasher():
""" """
#ignore rempty files # ignore rempty files
if fsize==0: if fsize == 0:
return False return False
# need to skip chunks? # need to skip chunks?
@ -53,7 +52,7 @@ class BlockHasher():
# seek to next chunk, reset skip count # seek to next chunk, reset skip count
fh.seek(self.chunk_size * self._skip_count, os.SEEK_CUR) fh.seek(self.chunk_size * self._skip_count, os.SEEK_CUR)
self._skip_count = self.skip self._skip_count = self.skip
return fh.tell()//self.chunk_size return fh.tell() // self.chunk_size
else: else:
# should read this chunk, reset skip count # should read this chunk, reset skip count
self._skip_count = self.skip self._skip_count = self.skip
@ -67,24 +66,23 @@ class BlockHasher():
yields nothing for empty files. yields nothing for empty files.
""" """
with open(fname, "rb") as fh: with open(fname, "rb") as fh:
fh.seek(0, os.SEEK_END) fh.seek(0, os.SEEK_END)
fsize=fh.tell() fsize = fh.tell()
fh.seek(0) fh.seek(0)
while fh.tell()<fsize: while fh.tell() < fsize:
chunk_nr=self._seek_next_chunk(fh, fsize) chunk_nr = self._seek_next_chunk(fh, fsize)
if chunk_nr is False: if chunk_nr is False:
return return
#read chunk # read chunk
hash = self.hash_class() hash = self.hash_class()
block_nr = 0 block_nr = 0
while block_nr != self.count: while block_nr != self.count:
block=fh.read(self.bs) block = fh.read(self.bs)
if block==b"": if block == b"":
break break
hash.update(block) hash.update(block)
block_nr = block_nr + 1 block_nr = block_nr + 1
@ -121,7 +119,7 @@ class BlockHasher():
yield (chunk_nr, hexdigest, hash.hexdigest()) yield (chunk_nr, hexdigest, hash.hexdigest())
except Exception as e: except Exception as e:
yield ( chunk_nr , hexdigest, 'ERROR: '+str(e)) yield (chunk_nr, hexdigest, 'ERROR: ' + str(e))
except Exception as e: except Exception as e:
yield ( '-', '-', 'ERROR: '+ str(e)) yield ('-', '-', 'ERROR: ' + str(e))

7
zfs_autobackup/CliBase.py
View File

@ -10,12 +10,12 @@ class CliBase(object):
Overridden in subclasses that add stuff for the specific programs.""" Overridden in subclasses that add stuff for the specific programs."""
# also used by setup.py # also used by setup.py
VERSION = "3.3-beta.3" VERSION = "3.4-beta.1"
HEADER = "{} v{} - (c)2022 E.H.Eefting (edwin@datux.nl)".format(os.path.basename(sys.argv[0]), VERSION) HEADER = "{} v{} - (c)2022 E.H.Eefting (edwin@datux.nl)".format(os.path.basename(sys.argv[0]), VERSION)
def __init__(self, argv, print_arguments=True): def __init__(self, argv, print_arguments=True):
self.parser=self.get_parser() self.parser = self.get_parser()
self.args = self.parse_args(argv) self.args = self.parse_args(argv)
# helps with investigating failed regression tests: # helps with investigating failed regression tests:
@ -66,7 +66,7 @@ class CliBase(object):
epilog='Full manual at: https://github.com/psy0rz/zfs_autobackup') epilog='Full manual at: https://github.com/psy0rz/zfs_autobackup')
# Basic options # Basic options
group=parser.add_argument_group("Common options") group = parser.add_argument_group("Common options")
group.add_argument('--help', '-h', action='store_true', help='show help') group.add_argument('--help', '-h', action='store_true', help='show help')
group.add_argument('--test', '--dry-run', '-n', action='store_true', group.add_argument('--test', '--dry-run', '-n', action='store_true',
help='Dry run, dont change anything, just show what would be done (still does all read-only ' help='Dry run, dont change anything, just show what would be done (still does all read-only '
@ -85,7 +85,6 @@ class CliBase(object):
group.add_argument('--version', action='store_true', group.add_argument('--version', action='store_true',
help='Show version.') help='Show version.')
return parser return parser
def verbose(self, txt): def verbose(self, txt):

7
zfs_autobackup/CmdPipe.py
View File

@ -41,7 +41,7 @@ class CmdItem:
self.exit_handler = exit_handler self.exit_handler = exit_handler
self.shell = shell self.shell = shell
self.process = None self.process = None
self.next = None #next item in pipe, set by CmdPipe self.next = None # next item in pipe, set by CmdPipe
def __str__(self): def __str__(self):
"""return copy-pastable version of command.""" """return copy-pastable version of command."""
@ -126,7 +126,7 @@ class CmdPipe:
success = True success = True
for item in self.items: for item in self.items:
if item.exit_handler is not None: if item.exit_handler is not None:
success=item.exit_handler(item.process.returncode) and success success = item.exit_handler(item.process.returncode) and success
return success return success
@ -159,7 +159,6 @@ class CmdPipe:
else: else:
eof_count = eof_count + 1 eof_count = eof_count + 1
if item.process.poll() is not None: if item.process.poll() is not None:
done_count = done_count + 1 done_count = done_count + 1
@ -167,8 +166,6 @@ class CmdPipe:
if eof_count == len(selectors) and done_count == len(self.items): if eof_count == len(selectors) and done_count == len(self.items):
break break
def __create(self): def __create(self):
"""create actual processes, do piping and return selectors.""" """create actual processes, do piping and return selectors."""

55
zfs_autobackup/ExecuteNode.py
View File

@ -17,7 +17,7 @@ class ExecuteError(Exception):
class ExecuteNode(LogStub): class ExecuteNode(LogStub):
"""an endpoint to execute local or remote commands via ssh""" """an endpoint to execute local or remote commands via ssh"""
PIPE=1 PIPE = 1
def __init__(self, ssh_config=None, ssh_to=None, readonly=False, debug_output=False): def __init__(self, ssh_config=None, ssh_to=None, readonly=False, debug_output=False):
"""ssh_config: custom ssh config """ssh_config: custom ssh config
@ -51,35 +51,35 @@ class ExecuteNode(LogStub):
def _quote(self, cmd): def _quote(self, cmd):
"""return quoted version of command. if it has value PIPE it will add an actual | """ """return quoted version of command. if it has value PIPE it will add an actual | """
if cmd==self.PIPE: if cmd == self.PIPE:
return('|') return ('|')
else: else:
return cmd_quote(cmd) return cmd_quote(cmd)
def _shell_cmd(self, cmd, cwd): def _shell_cmd(self, cmd, cwd):
"""prefix specified ssh shell to command and escape shell characters""" """prefix specified ssh shell to command and escape shell characters"""
ret=[] ret = []
#add remote shell # add remote shell
if not self.is_local(): if not self.is_local():
#note: dont escape this part (executed directly without shell) # note: dont escape this part (executed directly without shell)
ret=["ssh"] ret = ["ssh"]
if self.ssh_config is not None: if self.ssh_config is not None:
ret.extend(["-F", self.ssh_config]) ret.extend(["-F", self.ssh_config])
ret.append(self.ssh_to) ret.append(self.ssh_to)
#note: DO escape from here, executed in either local or remote shell. # note: DO escape from here, executed in either local or remote shell.
shell_str="" shell_str = ""
#add cwd change? # add cwd change?
if cwd is not None: if cwd is not None:
shell_str=shell_str + "cd " + self._quote(cwd) + "; " shell_str = shell_str + "cd " + self._quote(cwd) + "; "
shell_str=shell_str + " ".join(map(self._quote, cmd)) shell_str = shell_str + " ".join(map(self._quote, cmd))
ret.append(shell_str) ret.append(shell_str)
@ -121,7 +121,7 @@ class ExecuteNode(LogStub):
# stderr parser # stderr parser
error_lines = [] error_lines = []
returned_exit_code=None returned_exit_code = None
def stderr_handler(line): def stderr_handler(line):
if tab_split: if tab_split:
@ -139,7 +139,8 @@ class ExecuteNode(LogStub):
self.debug("EXIT > {}".format(exit_code)) self.debug("EXIT > {}".format(exit_code))
if (valid_exitcodes != []) and (exit_code not in valid_exitcodes): if (valid_exitcodes != []) and (exit_code not in valid_exitcodes):
self.error("Command \"{}\" returned exit code {} (valid codes: {})".format(cmd_item, exit_code, valid_exitcodes)) self.error("Command \"{}\" returned exit code {} (valid codes: {})".format(cmd_item, exit_code,
valid_exitcodes))
return False return False
return True return True
@ -149,7 +150,7 @@ class ExecuteNode(LogStub):
if pipe: if pipe:
# dont specify output handler, so it will get piped to next process # dont specify output handler, so it will get piped to next process
stdout_handler=None stdout_handler = None
else: else:
# handle output manually, dont pipe it # handle output manually, dont pipe it
def stdout_handler(line): def stdout_handler(line):
@ -160,7 +161,8 @@ class ExecuteNode(LogStub):
self._parse_stdout(line) self._parse_stdout(line)
# add shell command and handlers to pipe # add shell command and handlers to pipe
cmd_item=CmdItem(cmd=self._shell_cmd(cmd, cwd), readonly=readonly, stderr_handler=stderr_handler, exit_handler=exit_handler, shell=self.is_local(), stdout_handler=stdout_handler) cmd_item = CmdItem(cmd=self._shell_cmd(cmd, cwd), readonly=readonly, stderr_handler=stderr_handler,
exit_handler=exit_handler, shell=self.is_local(), stdout_handler=stdout_handler)
cmd_pipe.add(cmd_item) cmd_pipe.add(cmd_item)
# return CmdPipe instead of executing? # return CmdPipe instead of executing?
@ -174,7 +176,7 @@ class ExecuteNode(LogStub):
# execute and calls handlers in CmdPipe # execute and calls handlers in CmdPipe
if not cmd_pipe.execute(): if not cmd_pipe.execute():
raise(ExecuteError("Last command returned error")) raise (ExecuteError("Last command returned error"))
if return_all: if return_all:
return output_lines, error_lines, cmd_item.process and cmd_item.process.returncode return output_lines, error_lines, cmd_item.process and cmd_item.process.returncode
@ -183,7 +185,8 @@ class ExecuteNode(LogStub):
else: else:
return output_lines return output_lines
def script(self, lines, inp=None, stdout_handler=None, stderr_handler=None, exit_handler=None, valid_exitcodes=None, readonly=False, hide_errors=False, pipe=False): def script(self, lines, inp=None, stdout_handler=None, stderr_handler=None, exit_handler=None, valid_exitcodes=None,
readonly=False, hide_errors=False, pipe=False):
"""Run a multiline script on the node. """Run a multiline script on the node.
This is much more low level than run() and allows for finer grained control. This is much more low level than run() and allows for finer grained control.
@ -212,14 +215,14 @@ class ExecuteNode(LogStub):
# add stuff to existing pipe # add stuff to existing pipe
cmd_pipe = inp cmd_pipe = inp
internal_stdout_handler=None internal_stdout_handler = None
if stdout_handler is not None: if stdout_handler is not None:
if self.debug_output: if self.debug_output:
def internal_stdout_handler(line): def internal_stdout_handler(line):
self.debug("STDOUT > " + line.rstrip()) self.debug("STDOUT > " + line.rstrip())
stdout_handler(line) stdout_handler(line)
else: else:
internal_stdout_handler=stdout_handler internal_stdout_handler = stdout_handler
def internal_stderr_handler(line): def internal_stderr_handler(line):
self._parse_stderr(line, hide_errors) self._parse_stderr(line, hide_errors)
@ -243,12 +246,12 @@ class ExecuteNode(LogStub):
return True return True
#build command # build command
cmd=[] cmd = []
#add remote shell # add remote shell
if not self.is_local(): if not self.is_local():
#note: dont escape this part (executed directly without shell) # note: dont escape this part (executed directly without shell)
cmd.append("ssh") cmd.append("ssh")
if self.ssh_config is not None: if self.ssh_config is not None:
@ -260,7 +263,9 @@ class ExecuteNode(LogStub):
cmd.append("\n".join(lines)) cmd.append("\n".join(lines))
# add shell command and handlers to pipe # add shell command and handlers to pipe
cmd_item=CmdItem(cmd=cmd, readonly=readonly, stderr_handler=internal_stderr_handler, exit_handler=internal_exit_handler, stdout_handler=internal_stdout_handler, shell=self.is_local()) cmd_item = CmdItem(cmd=cmd, readonly=readonly, stderr_handler=internal_stderr_handler,
exit_handler=internal_exit_handler, stdout_handler=internal_stdout_handler,
shell=self.is_local())
cmd_pipe.add(cmd_item) cmd_pipe.add(cmd_item)
self.debug("SCRIPT > {}".format(cmd_pipe)) self.debug("SCRIPT > {}".format(cmd_pipe))

11
zfs_autobackup/LogConsole.py
View File

@ -3,6 +3,7 @@ from __future__ import print_function
import sys import sys
class LogConsole: class LogConsole:
"""Log-class that outputs to console, adding colors if needed""" """Log-class that outputs to console, adding colors if needed"""
@ -10,11 +11,11 @@ class LogConsole:
self.last_log = "" self.last_log = ""
self.show_debug = show_debug self.show_debug = show_debug
self.show_verbose = show_verbose self.show_verbose = show_verbose
self._progress_uncleared=False self._progress_uncleared = False
if color: if color:
# try to use color, failback if colorama not available # try to use color, failback if colorama not available
self.colorama=False self.colorama = False
try: try:
import colorama import colorama
global colorama global colorama
@ -23,7 +24,7 @@ class LogConsole:
pass pass
else: else:
self.colorama=False self.colorama = False
def error(self, txt): def error(self, txt):
self.clear_progress() self.clear_progress()
@ -62,7 +63,7 @@ class LogConsole:
def progress(self, txt): def progress(self, txt):
"""print progress output to stderr (stays on same line)""" """print progress output to stderr (stays on same line)"""
self.clear_progress() self.clear_progress()
self._progress_uncleared=True self._progress_uncleared = True
print(">>> {}\r".format(txt), end='', file=sys.stderr) print(">>> {}\r".format(txt), end='', file=sys.stderr)
sys.stderr.flush() sys.stderr.flush()
@ -71,4 +72,4 @@ class LogConsole:
import colorama import colorama
print(colorama.ansi.clear_line(), end='', file=sys.stderr) print(colorama.ansi.clear_line(), end='', file=sys.stderr)
# sys.stderr.flush() # sys.stderr.flush()
self._progress_uncleared=False self._progress_uncleared = False

4
zfs_autobackup/LogStub.py
View File

@ -1,5 +1,5 @@
#Used for baseclasses that dont implement their own logging (Like ExecuteNode) # Used for baseclasses that dont implement their own logging (Like ExecuteNode)
#Usually logging is implemented in subclasses (Like ZfsNode thats a subclass of ExecuteNode), but for regression testing its nice to have these stubs. # Usually logging is implemented in subclasses (Like ZfsNode thats a subclass of ExecuteNode), but for regression testing its nice to have these stubs.
class LogStub: class LogStub:
"""Just a stub, usually overriden in subclasses.""" """Just a stub, usually overriden in subclasses."""

4
zfs_autobackup/Thinner.py
View File

@ -1,4 +1,3 @@
from .ThinnerRule import ThinnerRule from .ThinnerRule import ThinnerRule
@ -48,7 +47,6 @@ class Thinner:
now: if specified, use this time as current time now: if specified, use this time as current time
""" """
# always keep a number of the last objets? # always keep a number of the last objets?
if self.always_keep: if self.always_keep:
# all of them # all of them
@ -71,7 +69,7 @@ class Thinner:
# traverse objects # traverse objects
for thisobject in objects: for thisobject in objects:
#ignore stuff without timestamp, always keep those. # ignore stuff without timestamp, always keep those.
if thisobject.timestamp is None: if thisobject.timestamp is None:
keeps.append(thisobject) keeps.append(thisobject)
else: else:

27
zfs_autobackup/TreeHasher.py
View File

@ -14,7 +14,7 @@ class TreeHasher():
:type block_hasher: BlockHasher :type block_hasher: BlockHasher
""" """
self.block_hasher=block_hasher self.block_hasher = block_hasher
def generate(self, start_path): def generate(self, start_path):
"""Use BlockHasher on every file in a tree, yielding the results """Use BlockHasher on every file in a tree, yielding the results
@ -28,12 +28,11 @@ class TreeHasher():
for (dirpath, dirnames, filenames) in os.walk(start_path, onerror=walkerror): for (dirpath, dirnames, filenames) in os.walk(start_path, onerror=walkerror):
for f in filenames: for f in filenames:
file_path=os.path.join(dirpath, f) file_path = os.path.join(dirpath, f)
if (not os.path.islink(file_path)) and os.path.isfile(file_path): if (not os.path.islink(file_path)) and os.path.isfile(file_path):
for (chunk_nr, hash) in self.block_hasher.generate(file_path): for (chunk_nr, hash) in self.block_hasher.generate(file_path):
yield ( os.path.relpath(file_path,start_path), chunk_nr, hash ) yield (os.path.relpath(file_path, start_path), chunk_nr, hash)
def compare(self, start_path, generator): def compare(self, start_path, generator):
"""reads from generator and compares blocks """reads from generator and compares blocks
@ -43,18 +42,14 @@ class TreeHasher():
""" """
count=0 count = 0
def filter_file_name( file_name, chunk_nr, hexdigest):
return ( chunk_nr, hexdigest )
def filter_file_name(file_name, chunk_nr, hexdigest):
return (chunk_nr, hexdigest)
for file_name, group_generator in itertools.groupby(generator, lambda x: x[0]): for file_name, group_generator in itertools.groupby(generator, lambda x: x[0]):
count=count+1 count = count + 1
block_generator=itertools.starmap(filter_file_name, group_generator) block_generator = itertools.starmap(filter_file_name, group_generator)
for ( chunk_nr, compare_hexdigest, actual_hexdigest) in self.block_hasher.compare(os.path.join(start_path,file_name), block_generator): for (chunk_nr, compare_hexdigest, actual_hexdigest) in self.block_hasher.compare(
yield ( file_name, chunk_nr, compare_hexdigest, actual_hexdigest ) os.path.join(start_path, file_name), block_generator):
yield (file_name, chunk_nr, compare_hexdigest, actual_hexdigest)

23
zfs_autobackup/ZfsAuto.py
View File

@ -49,13 +49,14 @@ class ZfsAuto(CliBase):
self.exclude_paths.append(args.target_path) self.exclude_paths.append(args.target_path)
else: else:
if not args.exclude_received and not args.include_received: if not args.exclude_received and not args.include_received:
self.verbose("NOTE: Source and target are on the same host, adding --exclude-received to commandline. (use --include-received to overrule)") self.verbose(
"NOTE: Source and target are on the same host, adding --exclude-received to commandline. (use --include-received to overrule)")
args.exclude_received = True args.exclude_received = True
if args.test: if args.test:
self.warning("TEST MODE - SIMULATING WITHOUT MAKING ANY CHANGES") self.warning("TEST MODE - SIMULATING WITHOUT MAKING ANY CHANGES")
#format all the names # format all the names
self.property_name = args.property_format.format(args.backup_name) self.property_name = args.property_format.format(args.backup_name)
self.snapshot_time_format = args.snapshot_format.format(args.backup_name) self.snapshot_time_format = args.snapshot_format.format(args.backup_name)
self.hold_name = args.hold_format.format(args.backup_name) self.hold_name = args.hold_format.format(args.backup_name)
@ -63,7 +64,8 @@ class ZfsAuto(CliBase):
dt = datetime_now(args.utc) dt = datetime_now(args.utc)
self.verbose("") self.verbose("")
self.verbose("Current time {} : {}".format(args.utc and "UTC" or " ", dt.strftime("%Y-%m-%d %H:%M:%S"))) self.verbose(
"Current time {} : {}".format(args.utc and "UTC" or " ", dt.strftime("%Y-%m-%d %H:%M:%S")))
self.verbose("Selecting dataset property : {}".format(self.property_name)) self.verbose("Selecting dataset property : {}".format(self.property_name))
self.verbose("Snapshot format : {}".format(self.snapshot_time_format)) self.verbose("Snapshot format : {}".format(self.snapshot_time_format))
@ -75,24 +77,22 @@ class ZfsAuto(CliBase):
parser = super(ZfsAuto, self).get_parser() parser = super(ZfsAuto, self).get_parser()
#positional arguments # positional arguments
parser.add_argument('backup_name', metavar='BACKUP-NAME', default=None, nargs='?', parser.add_argument('backup_name', metavar='BACKUP-NAME', default=None, nargs='?',
help='Name of the backup to select') help='Name of the backup to select')
parser.add_argument('target_path', metavar='TARGET-PATH', default=None, nargs='?', parser.add_argument('target_path', metavar='TARGET-PATH', default=None, nargs='?',
help='Target ZFS filesystem (optional)') help='Target ZFS filesystem (optional)')
# SSH options # SSH options
group=parser.add_argument_group("SSH options") group = parser.add_argument_group("SSH options")
group.add_argument('--ssh-config', metavar='CONFIG-FILE', default=None, help='Custom ssh client config') group.add_argument('--ssh-config', metavar='CONFIG-FILE', default=None, help='Custom ssh client config')
group.add_argument('--ssh-source', metavar='USER@HOST', default=None, group.add_argument('--ssh-source', metavar='USER@HOST', default=None,
help='Source host to pull backup from.') help='Source host to pull backup from.')
group.add_argument('--ssh-target', metavar='USER@HOST', default=None, group.add_argument('--ssh-target', metavar='USER@HOST', default=None,
help='Target host to push backup to.') help='Target host to push backup to.')
group=parser.add_argument_group("String formatting options") group = parser.add_argument_group("String formatting options")
group.add_argument('--property-format', metavar='FORMAT', default="autobackup:{}", group.add_argument('--property-format', metavar='FORMAT', default="autobackup:{}",
help='Dataset selection string format. Default: %(default)s') help='Dataset selection string format. Default: %(default)s')
group.add_argument('--snapshot-format', metavar='FORMAT', default="{}-%Y%m%d%H%M%S", group.add_argument('--snapshot-format', metavar='FORMAT', default="{}-%Y%m%d%H%M%S",
@ -102,7 +102,7 @@ class ZfsAuto(CliBase):
group.add_argument('--strip-path', metavar='N', default=0, type=int, group.add_argument('--strip-path', metavar='N', default=0, type=int,
help='Number of directories to strip from target path.') help='Number of directories to strip from target path.')
group=parser.add_argument_group("Selection options") group = parser.add_argument_group("Selection options")
group.add_argument('--ignore-replicated', action='store_true', help=argparse.SUPPRESS) group.add_argument('--ignore-replicated', action='store_true', help=argparse.SUPPRESS)
group.add_argument('--exclude-unchanged', metavar='BYTES', default=0, type=int, group.add_argument('--exclude-unchanged', metavar='BYTES', default=0, type=int,
help='Exclude datasets that have less than BYTES data changed since any last snapshot. (Use with proxmox HA replication)') help='Exclude datasets that have less than BYTES data changed since any last snapshot. (Use with proxmox HA replication)')
@ -112,14 +112,15 @@ class ZfsAuto(CliBase):
group.add_argument('--include-received', action='store_true', group.add_argument('--include-received', action='store_true',
help=argparse.SUPPRESS) help=argparse.SUPPRESS)
def regex_argument_type(input_line): def regex_argument_type(input_line):
"""Parses regex arguments into re.Pattern objects""" """Parses regex arguments into re.Pattern objects"""
try: try:
return re.compile(input_line) return re.compile(input_line)
except: except:
raise ValueError("Could not parse argument '{}' as a regular expression".format(input_line)) raise ValueError("Could not parse argument '{}' as a regular expression".format(input_line))
group.add_argument('--exclude-snapshot-pattern', action='append', default=[], type=regex_argument_type, help="Regular expression to match snapshots that will be ignored.")
group.add_argument('--exclude-snapshot-pattern', action='append', default=[], type=regex_argument_type,
help="Regular expression to match snapshots that will be ignored.")
return parser return parser

25
zfs_autobackup/ZfsAutobackup.py
View File

@ -1,4 +1,3 @@
import argparse import argparse
from signal import signal, SIGPIPE from signal import signal, SIGPIPE
from .util import output_redir, sigpipe_handler, datetime_now from .util import output_redir, sigpipe_handler, datetime_now
@ -12,6 +11,7 @@ from .ZfsDataset import ZfsDataset
from .ZfsNode import ZfsNode from .ZfsNode import ZfsNode
from .ThinnerRule import ThinnerRule from .ThinnerRule import ThinnerRule
class ZfsAutobackup(ZfsAuto): class ZfsAutobackup(ZfsAuto):
"""The main zfs-autobackup class. Start here, at run() :)""" """The main zfs-autobackup class. Start here, at run() :)"""
@ -73,7 +73,6 @@ class ZfsAutobackup(ZfsAuto):
group.add_argument('--no-guid-check', action='store_true', group.add_argument('--no-guid-check', action='store_true',
help='Dont check guid of common snapshots. (faster)') help='Dont check guid of common snapshots. (faster)')
group = parser.add_argument_group("Transfer options") group = parser.add_argument_group("Transfer options")
group.add_argument('--no-send', action='store_true', group.add_argument('--no-send', action='store_true',
help='Don\'t transfer snapshots (useful for cleanups, or if you want a separate send-cronjob)') help='Don\'t transfer snapshots (useful for cleanups, or if you want a separate send-cronjob)')
@ -324,8 +323,8 @@ class ZfsAutobackup(ZfsAuto):
def make_target_name(self, source_dataset): def make_target_name(self, source_dataset):
"""make target_name from a source_dataset""" """make target_name from a source_dataset"""
stripped=source_dataset.lstrip_path(self.args.strip_path) stripped = source_dataset.lstrip_path(self.args.strip_path)
if stripped!="": if stripped != "":
return self.args.target_path + "/" + stripped return self.args.target_path + "/" + stripped
else: else:
return self.args.target_path return self.args.target_path
@ -334,16 +333,20 @@ class ZfsAutobackup(ZfsAuto):
"""check all target names for collesions etc due to strip-options""" """check all target names for collesions etc due to strip-options"""
self.debug("Checking target names:") self.debug("Checking target names:")
target_datasets={} target_datasets = {}
for source_dataset in source_datasets: for source_dataset in source_datasets:
target_name = self.make_target_name(source_dataset) target_name = self.make_target_name(source_dataset)
source_dataset.debug("-> {}".format(target_name)) source_dataset.debug("-> {}".format(target_name))
if target_name in target_datasets: if target_name in target_datasets:
raise Exception("Target collision: Target path {} encountered twice, due to: {} and {}".format(target_name, source_dataset, target_datasets[target_name])) raise Exception(
"Target collision: Target path {} encountered twice, due to: {} and {}".format(target_name,
source_dataset,
target_datasets[
target_name]))
target_datasets[target_name]=source_dataset target_datasets[target_name] = source_dataset
# NOTE: this method also uses self.args. args that need extra processing are passed as function parameters: # NOTE: this method also uses self.args. args that need extra processing are passed as function parameters:
def sync_datasets(self, source_node, source_datasets, target_node): def sync_datasets(self, source_node, source_datasets, target_node):
@ -397,7 +400,8 @@ class ZfsAutobackup(ZfsAuto):
destroy_incompatible=self.args.destroy_incompatible, destroy_incompatible=self.args.destroy_incompatible,
send_pipes=send_pipes, recv_pipes=recv_pipes, send_pipes=send_pipes, recv_pipes=recv_pipes,
decrypt=self.args.decrypt, encrypt=self.args.encrypt, decrypt=self.args.decrypt, encrypt=self.args.encrypt,
zfs_compressed=self.args.zfs_compressed, force=self.args.force, guid_check=not self.args.no_guid_check) zfs_compressed=self.args.zfs_compressed, force=self.args.force,
guid_check=not self.args.no_guid_check)
except Exception as e: except Exception as e:
fail_count = fail_count + 1 fail_count = fail_count + 1
@ -406,7 +410,6 @@ class ZfsAutobackup(ZfsAuto):
self.verbose("Debug mode, aborting on first error") self.verbose("Debug mode, aborting on first error")
raise raise
target_path_dataset = target_node.get_dataset(self.args.target_path) target_path_dataset = target_node.get_dataset(self.args.target_path)
if not self.args.no_thinning: if not self.args.no_thinning:
self.thin_missing_targets(target_dataset=target_path_dataset, used_target_datasets=target_datasets) self.thin_missing_targets(target_dataset=target_path_dataset, used_target_datasets=target_datasets)
@ -477,7 +480,7 @@ class ZfsAutobackup(ZfsAuto):
################# select source datasets ################# select source datasets
self.set_title("Selecting") self.set_title("Selecting")
( source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, (source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name,
exclude_received=self.args.exclude_received, exclude_received=self.args.exclude_received,
exclude_paths=self.exclude_paths, exclude_paths=self.exclude_paths,
exclude_unchanged=self.args.exclude_unchanged) exclude_unchanged=self.args.exclude_unchanged)
@ -572,7 +575,7 @@ def cli():
signal(SIGPIPE, sigpipe_handler) signal(SIGPIPE, sigpipe_handler)
failed_datasets=ZfsAutobackup(sys.argv[1:], False).run() failed_datasets = ZfsAutobackup(sys.argv[1:], False).run()
sys.exit(min(failed_datasets, 255)) sys.exit(min(failed_datasets, 255))

51
zfs_autobackup/ZfsAutoverify.py
View File

@ -76,14 +76,14 @@ def verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt,
source_snapshot.mount(source_mnt) source_snapshot.mount(source_mnt)
target_snapshot.mount(target_mnt) target_snapshot.mount(target_mnt)
if method=='rsync': if method == 'rsync':
compare_trees_rsync(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt) compare_trees_rsync(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt)
# elif method == 'tar': # elif method == 'tar':
# compare_trees_tar(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt) # compare_trees_tar(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt)
elif method == 'find': elif method == 'find':
compare_trees_find(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt) compare_trees_find(source_snapshot.zfs_node, source_mnt, target_snapshot.zfs_node, target_mnt)
else: else:
raise(Exception("program errror, unknown method")) raise (Exception("program errror, unknown method"))
finally: finally:
source_snapshot.unmount(source_mnt) source_snapshot.unmount(source_mnt)
@ -109,7 +109,6 @@ def verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt,
# return hashed # return hashed
# def deacitvate_volume_snapshot(snapshot): # def deacitvate_volume_snapshot(snapshot):
# clone_name=get_tmp_clone_name(snapshot) # clone_name=get_tmp_clone_name(snapshot)
# clone=snapshot.zfs_node.get_dataset(clone_name) # clone=snapshot.zfs_node.get_dataset(clone_name)
@ -119,13 +118,13 @@ def verify_volume(source_dataset, source_snapshot, target_dataset, target_snapsh
"""compare the contents of two zfs volume snapshots""" """compare the contents of two zfs volume snapshots"""
# try: # try:
source_dev= activate_volume_snapshot(source_snapshot) source_dev = activate_volume_snapshot(source_snapshot)
target_dev= activate_volume_snapshot(target_snapshot) target_dev = activate_volume_snapshot(target_snapshot)
source_hash= hash_dev(source_snapshot.zfs_node, source_dev) source_hash = hash_dev(source_snapshot.zfs_node, source_dev)
target_hash= hash_dev(target_snapshot.zfs_node, target_dev) target_hash = hash_dev(target_snapshot.zfs_node, target_dev)
if source_hash!=target_hash: if source_hash != target_hash:
raise Exception("md5hash difference: {} != {}".format(source_hash, target_hash)) raise Exception("md5hash difference: {} != {}".format(source_hash, target_hash))
# finally: # finally:
@ -150,7 +149,7 @@ class ZfsAutoverify(ZfsAuto):
def parse_args(self, argv): def parse_args(self, argv):
"""do extra checks on common args""" """do extra checks on common args"""
args=super(ZfsAutoverify, self).parse_args(argv) args = super(ZfsAutoverify, self).parse_args(argv)
if args.target_path == None: if args.target_path == None:
self.log.error("Please specify TARGET-PATH") self.log.error("Please specify TARGET-PATH")
@ -161,9 +160,9 @@ class ZfsAutoverify(ZfsAuto):
def get_parser(self): def get_parser(self):
"""extend common parser with extra stuff needed for zfs-autobackup""" """extend common parser with extra stuff needed for zfs-autobackup"""
parser=super(ZfsAutoverify, self).get_parser() parser = super(ZfsAutoverify, self).get_parser()
group=parser.add_argument_group("Verify options") group = parser.add_argument_group("Verify options")
group.add_argument('--fs-compare', metavar='METHOD', default="find", choices=["find", "rsync"], group.add_argument('--fs-compare', metavar='METHOD', default="find", choices=["find", "rsync"],
help='Compare method to use for filesystems. (find, rsync) Default: %(default)s ') help='Compare method to use for filesystems. (find, rsync) Default: %(default)s ')
@ -171,7 +170,7 @@ class ZfsAutoverify(ZfsAuto):
def verify_datasets(self, source_mnt, source_datasets, target_node, target_mnt): def verify_datasets(self, source_mnt, source_datasets, target_node, target_mnt):
fail_count=0 fail_count = 0
count = 0 count = 0
for source_dataset in source_datasets: for source_dataset in source_datasets:
@ -190,16 +189,17 @@ class ZfsAutoverify(ZfsAuto):
target_snapshot = target_dataset.find_snapshot(source_snapshot) target_snapshot = target_dataset.find_snapshot(source_snapshot)
if source_snapshot is None or target_snapshot is None: if source_snapshot is None or target_snapshot is None:
raise(Exception("Cant find common snapshot")) raise (Exception("Cant find common snapshot"))
target_snapshot.verbose("Verifying...") target_snapshot.verbose("Verifying...")
if source_dataset.properties['type']=="filesystem": if source_dataset.properties['type'] == "filesystem":
verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt, self.args.fs_compare) verify_filesystem(source_snapshot, source_mnt, target_snapshot, target_mnt, self.args.fs_compare)
elif source_dataset.properties['type']=="volume": elif source_dataset.properties['type'] == "volume":
verify_volume(source_dataset, source_snapshot, target_dataset, target_snapshot) verify_volume(source_dataset, source_snapshot, target_dataset, target_snapshot)
else: else:
raise(Exception("{} has unknown type {}".format(source_dataset, source_dataset.properties['type']))) raise (
Exception("{} has unknown type {}".format(source_dataset, source_dataset.properties['type'])))
except Exception as e: except Exception as e:
@ -219,11 +219,10 @@ class ZfsAutoverify(ZfsAuto):
def run(self): def run(self):
source_node=None source_node = None
source_mnt=None source_mnt = None
target_node=None target_node = None
target_mnt=None target_mnt = None
try: try:
@ -240,7 +239,7 @@ class ZfsAutoverify(ZfsAuto):
################# select source datasets ################# select source datasets
self.set_title("Selecting") self.set_title("Selecting")
( source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name, (source_datasets, excluded_datasets) = source_node.selected_datasets(property_name=self.property_name,
exclude_received=self.args.exclude_received, exclude_received=self.args.exclude_received,
exclude_paths=self.exclude_paths, exclude_paths=self.exclude_paths,
exclude_unchanged=self.args.exclude_unchanged) exclude_unchanged=self.args.exclude_unchanged)
@ -260,7 +259,7 @@ class ZfsAutoverify(ZfsAuto):
self.set_title("Verifying") self.set_title("Verifying")
source_mnt, target_mnt= create_mountpoints(source_node, target_node) source_mnt, target_mnt = create_mountpoints(source_node, target_node)
fail_count = self.verify_datasets( fail_count = self.verify_datasets(
source_mnt=source_mnt, source_mnt=source_mnt,
@ -302,15 +301,13 @@ class ZfsAutoverify(ZfsAuto):
cleanup_mountpoint(target_node, target_mnt) cleanup_mountpoint(target_node, target_mnt)
def cli(): def cli():
import sys import sys
raise(Exception("This program is incomplete, dont use it yet.")) raise (Exception("This program is incomplete, dont use it yet."))
signal(SIGPIPE, sigpipe_handler) signal(SIGPIPE, sigpipe_handler)
failed = ZfsAutoverify(sys.argv[1:], False).run() failed = ZfsAutoverify(sys.argv[1:], False).run()
sys.exit(min(failed,255)) sys.exit(min(failed, 255))
if __name__ == "__main__": if __name__ == "__main__":

75
zfs_autobackup/ZfsCheck.py
View File

@ -27,14 +27,16 @@ class ZfsCheck(CliBase):
parser = super(ZfsCheck, self).get_parser() parser = super(ZfsCheck, self).get_parser()
# positional arguments # positional arguments
parser.add_argument('target', metavar='TARGET', default=None, nargs='?', help='Target to checkum. (can be blockdevice, directory or ZFS snapshot)') parser.add_argument('target', metavar='TARGET', default=None, nargs='?',
help='Target to checkum. (can be blockdevice, directory or ZFS snapshot)')
group = parser.add_argument_group('Checker options') group = parser.add_argument_group('Checker options')
group.add_argument('--block-size', metavar="BYTES", default=4096, help="Read block-size, default %(default)s", group.add_argument('--block-size', metavar="BYTES", default=4096, help="Read block-size, default %(default)s",
type=int) type=int)
group.add_argument('--count', metavar="COUNT", default=int((100 * (1024 ** 2)) / 4096), group.add_argument('--count', metavar="COUNT", default=int((100 * (1024 ** 2)) / 4096),
help="Hash chunks of COUNT blocks. Default %(default)s . (CHUNK size is BYTES * COUNT) ", type=int) # 100MiB help="Hash chunks of COUNT blocks. Default %(default)s . (CHUNK size is BYTES * COUNT) ",
type=int) # 100MiB
group.add_argument('--check', '-c', metavar="FILE", default=None, const=True, nargs='?', group.add_argument('--check', '-c', metavar="FILE", default=None, const=True, nargs='?',
help="Read hashes from STDIN (or FILE) and compare them") help="Read hashes from STDIN (or FILE) and compare them")
@ -57,11 +59,10 @@ class ZfsCheck(CliBase):
self.verbose("Target : {}".format(args.target)) self.verbose("Target : {}".format(args.target))
self.verbose("Block size : {} bytes".format(args.block_size)) self.verbose("Block size : {} bytes".format(args.block_size))
self.verbose("Block count : {}".format(args.count)) self.verbose("Block count : {}".format(args.count))
self.verbose("Effective chunk size : {} bytes".format(args.count*args.block_size)) self.verbose("Effective chunk size : {} bytes".format(args.count * args.block_size))
self.verbose("Skip chunk count : {} (checks {:.2f}% of data)".format(args.skip, 100/(1+args.skip))) self.verbose("Skip chunk count : {} (checks {:.2f}% of data)".format(args.skip, 100 / (1 + args.skip)))
self.verbose("") self.verbose("")
return args return args
def prepare_zfs_filesystem(self, snapshot): def prepare_zfs_filesystem(self, snapshot):
@ -106,7 +107,9 @@ class ZfsCheck(CliBase):
time.sleep(1) time.sleep(1)
raise (Exception("Timeout while waiting for /dev entry to appear. (looking in: {}). Hint: did you forget to load the encryption key?".format(locations))) raise (Exception(
"Timeout while waiting for /dev entry to appear. (looking in: {}). Hint: did you forget to load the encryption key?".format(
locations)))
def cleanup_zfs_volume(self, snapshot): def cleanup_zfs_volume(self, snapshot):
"""destroys temporary volume snapshot""" """destroys temporary volume snapshot"""
@ -144,34 +147,34 @@ class ZfsCheck(CliBase):
"""parse input lines and yield items to use in compare functions""" """parse input lines and yield items to use in compare functions"""
if self.args.check is True: if self.args.check is True:
input_fh=sys.stdin input_fh = sys.stdin
else: else:
input_fh=open(self.args.check, 'r') input_fh = open(self.args.check, 'r')
last_progress_time = time.time() last_progress_time = time.time()
progress_checked = 0 progress_checked = 0
progress_skipped = 0 progress_skipped = 0
line=input_fh.readline() line = input_fh.readline()
skip=0 skip = 0
while line: while line:
i=line.rstrip().split("\t") i = line.rstrip().split("\t")
#ignores lines without tabs # ignores lines without tabs
if (len(i)>1): if (len(i) > 1):
if skip==0: if skip == 0:
progress_checked=progress_checked+1 progress_checked = progress_checked + 1
yield i yield i
skip=self.args.skip skip = self.args.skip
else: else:
skip=skip-1 skip = skip - 1
progress_skipped=progress_skipped+1 progress_skipped = progress_skipped + 1
if self.args.progress and time.time() - last_progress_time > 1: if self.args.progress and time.time() - last_progress_time > 1:
last_progress_time = time.time() last_progress_time = time.time()
self.progress("Checked {} hashes (skipped {})".format(progress_checked, progress_skipped)) self.progress("Checked {} hashes (skipped {})".format(progress_checked, progress_skipped))
line=input_fh.readline() line = input_fh.readline()
self.verbose("Checked {} hashes (skipped {})".format(progress_checked, progress_skipped)) self.verbose("Checked {} hashes (skipped {})".format(progress_checked, progress_skipped))
@ -224,7 +227,7 @@ class ZfsCheck(CliBase):
if "@" in self.args.target: if "@" in self.args.target:
# zfs snapshot # zfs snapshot
snapshot=self.node.get_dataset(self.args.target) snapshot = self.node.get_dataset(self.args.target)
if not snapshot.exists: if not snapshot.exists:
raise Exception("ZFS snapshot {} does not exist!".format(snapshot)) raise Exception("ZFS snapshot {} does not exist!".format(snapshot))
dataset_type = snapshot.parent.properties['type'] dataset_type = snapshot.parent.properties['type']
@ -240,7 +243,7 @@ class ZfsCheck(CliBase):
def cleanup_target(self): def cleanup_target(self):
if "@" in self.args.target: if "@" in self.args.target:
# zfs snapshot # zfs snapshot
snapshot=self.node.get_dataset(self.args.target) snapshot = self.node.get_dataset(self.args.target)
if not snapshot.exists: if not snapshot.exists:
return return
@ -253,28 +256,28 @@ class ZfsCheck(CliBase):
def run(self): def run(self):
compare_generator=None compare_generator = None
hash_generator=None hash_generator = None
try: try:
prepared_target=self.prepare_target() prepared_target = self.prepare_target()
is_dir=os.path.isdir(prepared_target) is_dir = os.path.isdir(prepared_target)
#run as compare # run as compare
if self.args.check is not None: if self.args.check is not None:
input_generator=self.generate_input() input_generator = self.generate_input()
if is_dir: if is_dir:
compare_generator = self.generate_tree_compare(prepared_target, input_generator) compare_generator = self.generate_tree_compare(prepared_target, input_generator)
else: else:
compare_generator=self.generate_file_compare(prepared_target, input_generator) compare_generator = self.generate_file_compare(prepared_target, input_generator)
errors=self.print_errors(compare_generator) errors = self.print_errors(compare_generator)
#run as generator # run as generator
else: else:
if is_dir: if is_dir:
hash_generator = self.generate_tree_hashes(prepared_target) hash_generator = self.generate_tree_hashes(prepared_target)
else: else:
hash_generator=self.generate_file_hashes(prepared_target) hash_generator = self.generate_file_hashes(prepared_target)
errors=self.print_hashes(hash_generator) errors = self.print_hashes(hash_generator)
except Exception as e: except Exception as e:
self.error("Exception: " + str(e)) self.error("Exception: " + str(e))
@ -286,10 +289,10 @@ class ZfsCheck(CliBase):
return 255 return 255
finally: finally:
#important to call check_output so that cleanup still functions in case of a broken pipe: # important to call check_output so that cleanup still functions in case of a broken pipe:
# util.check_output() # util.check_output()
#close generators, to make sure files are not in use anymore when cleaning up # close generators, to make sure files are not in use anymore when cleaning up
if hash_generator is not None: if hash_generator is not None:
hash_generator.close() hash_generator.close()
if compare_generator is not None: if compare_generator is not None:
@ -302,8 +305,8 @@ class ZfsCheck(CliBase):
def cli(): def cli():
import sys import sys
signal(SIGPIPE, sigpipe_handler) signal(SIGPIPE, sigpipe_handler)
failed=ZfsCheck(sys.argv[1:], False).run() failed = ZfsCheck(sys.argv[1:], False).run()
sys.exit(min(failed,255)) sys.exit(min(failed, 255))
if __name__ == "__main__": if __name__ == "__main__":

96
zfs_autobackup/ZfsDataset.py
View File

@ -1,4 +1,3 @@
import re import re
from datetime import datetime from datetime import datetime
import sys import sys
@ -29,28 +28,27 @@ class ZfsDataset:
self.zfs_node = zfs_node self.zfs_node = zfs_node
self.name = name # full name self.name = name # full name
#caching # caching
self.__snapshots=None #type: None|list[ZfsDataset] self.__snapshots = None # type: None|list[ZfsDataset]
self.__written_since_ours=None #type: None|int self.__written_since_ours = None # type: None|int
self.__exists_check=None #type: None|bool self.__exists_check = None # type: None|bool
self.__properties=None #type: None|dict[str,str] self.__properties = None # type: None|dict[str,str]
self.__recursive_datasets=None #type: None|list[ZfsDataset] self.__recursive_datasets = None # type: None|list[ZfsDataset]
self.__datasets=None #type: None|list[ZfsDataset] self.__datasets = None # type: None|list[ZfsDataset]
self.invalidate_cache() self.invalidate_cache()
self.force_exists = force_exists self.force_exists = force_exists
def invalidate_cache(self): def invalidate_cache(self):
"""clear caches""" """clear caches"""
# CachedProperty.clear(self) # CachedProperty.clear(self)
self.force_exists = None self.force_exists = None
self.__snapshots=None self.__snapshots = None
self.__written_since_ours=None self.__written_since_ours = None
self.__exists_check=None self.__exists_check = None
self.__properties=None self.__properties = None
self.__recursive_datasets=None self.__recursive_datasets = None
self.__datasets=None self.__datasets = None
def __repr__(self): def __repr__(self):
return "{}: {}".format(self.zfs_node, self.name) return "{}: {}".format(self.zfs_node, self.name)
@ -93,7 +91,6 @@ class ZfsDataset:
""" """
self.zfs_node.debug("{}: {}".format(self.name, txt)) self.zfs_node.debug("{}: {}".format(self.name, txt))
def split_path(self): def split_path(self):
"""return the path elements as an array""" """return the path elements as an array"""
return self.name.split("/") return self.name.split("/")
@ -147,14 +144,11 @@ class ZfsDataset:
if not self.is_snapshot: if not self.is_snapshot:
return False return False
for pattern in self.zfs_node.exclude_snapshot_patterns: for pattern in self.zfs_node.exclude_snapshot_patterns:
if pattern.search(self.name) is not None: if pattern.search(self.name) is not None:
self.debug("Excluded (path matches snapshot exclude pattern)") self.debug("Excluded (path matches snapshot exclude pattern)")
return True return True
def is_selected(self, value, source, inherited, exclude_received, exclude_paths, exclude_unchanged): def is_selected(self, value, source, inherited, exclude_received, exclude_paths, exclude_unchanged):
"""determine if dataset should be selected for backup (called from """determine if dataset should be selected for backup (called from
ZfsNode) ZfsNode)
@ -290,7 +284,7 @@ class ZfsDataset:
if self.__exists_check is None: if self.__exists_check is None:
self.debug("Checking if dataset exists") self.debug("Checking if dataset exists")
self.__exists_check=(len(self.zfs_node.run(tab_split=True, cmd=["zfs", "list", self.name], readonly=True, self.__exists_check = (len(self.zfs_node.run(tab_split=True, cmd=["zfs", "list", self.name], readonly=True,
valid_exitcodes=[0, 1], valid_exitcodes=[0, 1],
hide_errors=True)) > 0) hide_errors=True)) > 0)
@ -454,7 +448,6 @@ class ZfsDataset:
seconds = time.mktime(dt.timetuple()) seconds = time.mktime(dt.timetuple())
return seconds return seconds
# def add_virtual_snapshot(self, snapshot): # def add_virtual_snapshot(self, snapshot):
# """pretend a snapshot exists (usefull in test mode)""" # """pretend a snapshot exists (usefull in test mode)"""
# #
@ -474,18 +467,15 @@ class ZfsDataset:
:rtype: list[ZfsDataset] :rtype: list[ZfsDataset]
""" """
#cached? # cached?
if self.__snapshots is None: if self.__snapshots is None:
self.debug("Getting snapshots") self.debug("Getting snapshots")
cmd = [ cmd = [
"zfs", "list", "-d", "1", "-r", "-t", "snapshot", "-H", "-o", "name", self.name "zfs", "list", "-d", "1", "-r", "-t", "snapshot", "-H", "-o", "name", self.name
] ]
self.__snapshots=self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True) self.__snapshots = self.zfs_node.get_datasets(self.zfs_node.run(cmd=cmd, readonly=True), force_exists=True)
return self.__snapshots return self.__snapshots
@ -517,7 +507,7 @@ class ZfsDataset:
""" """
for snapshot in snapshots: for snapshot in snapshots:
if snapshot.snapshot_name==self.snapshot_name: if snapshot.snapshot_name == self.snapshot_name:
return snapshot return snapshot
return None return None
@ -571,7 +561,6 @@ class ZfsDataset:
"""get number of bytes written since our last snapshot""" """get number of bytes written since our last snapshot"""
if self.__written_since_ours is None: if self.__written_since_ours is None:
latest_snapshot = self.our_snapshots[-1] latest_snapshot = self.our_snapshots[-1]
self.debug("Getting bytes written since our last snapshot") self.debug("Getting bytes written since our last snapshot")
@ -579,7 +568,7 @@ class ZfsDataset:
output = self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0]) output = self.zfs_node.run(readonly=True, tab_split=False, cmd=cmd, valid_exitcodes=[0])
self.__written_since_ours=int(output[0]) self.__written_since_ours = int(output[0])
return self.__written_since_ours return self.__written_since_ours
@ -612,14 +601,13 @@ class ZfsDataset:
""" """
if self.__recursive_datasets is None: if self.__recursive_datasets is None:
self.debug("Getting all recursive datasets under us") self.debug("Getting all recursive datasets under us")
names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[
"zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name "zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name
]) ])
self.__recursive_datasets=self.zfs_node.get_datasets(names[1:], force_exists=True) self.__recursive_datasets = self.zfs_node.get_datasets(names[1:], force_exists=True)
return self.__recursive_datasets return self.__recursive_datasets
@ -632,14 +620,13 @@ class ZfsDataset:
""" """
if self.__datasets is None: if self.__datasets is None:
self.debug("Getting all datasets under us") self.debug("Getting all datasets under us")
names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[ names = self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[0], cmd=[
"zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name "zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name
]) ])
self.__datasets=self.zfs_node.get_datasets(names[1:], force_exists=True) self.__datasets = self.zfs_node.get_datasets(names[1:], force_exists=True)
return self.__datasets return self.__datasets
@ -804,7 +791,7 @@ class ZfsDataset:
if self.properties['encryption'] != 'off' and self.properties['keystatus'] == 'unavailable': if self.properties['encryption'] != 'off' and self.properties['keystatus'] == 'unavailable':
return return
self.zfs_node.run(["zfs", "mount", self.name], valid_exitcodes=[0,1]) self.zfs_node.run(["zfs", "mount", self.name], valid_exitcodes=[0, 1])
def transfer_snapshot(self, target_snapshot, features, prev_snapshot, show_progress, def transfer_snapshot(self, target_snapshot, features, prev_snapshot, show_progress,
filter_properties, set_properties, ignore_recv_exit_code, resume_token, filter_properties, set_properties, ignore_recv_exit_code, resume_token,
@ -960,7 +947,6 @@ class ZfsDataset:
# target_dataset.error("Cant find common snapshot with source.") # target_dataset.error("Cant find common snapshot with source.")
raise (Exception("Cant find common snapshot with target.")) raise (Exception("Cant find common snapshot with target."))
def find_incompatible_snapshots(self, common_snapshot, raw): def find_incompatible_snapshots(self, common_snapshot, raw):
"""returns a list[snapshots] that is incompatible for a zfs recv onto """returns a list[snapshots] that is incompatible for a zfs recv onto
the common_snapshot. all direct followup snapshots with written=0 are the common_snapshot. all direct followup snapshots with written=0 are
@ -1006,7 +992,6 @@ class ZfsDataset:
return allowed_filter_properties, allowed_set_properties return allowed_filter_properties, allowed_set_properties
def _pre_clean(self, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers): def _pre_clean(self, source_common_snapshot, target_dataset, source_obsoletes, target_obsoletes, target_transfers):
"""cleanup old stuff before starting snapshot syncing """cleanup old stuff before starting snapshot syncing
@ -1021,7 +1006,7 @@ class ZfsDataset:
# on source: delete all obsoletes that are not in target_transfers (except common snapshot) # on source: delete all obsoletes that are not in target_transfers (except common snapshot)
for source_snapshot in self.snapshots: for source_snapshot in self.snapshots:
if (source_snapshot in source_obsoletes if (source_snapshot in source_obsoletes
and source_common_snapshot!=source_snapshot and source_common_snapshot != source_snapshot
and source_snapshot.find_snapshot_in_list(target_transfers) is None): and source_snapshot.find_snapshot_in_list(target_transfers) is None):
source_snapshot.destroy() source_snapshot.destroy()
@ -1029,7 +1014,8 @@ class ZfsDataset:
if target_dataset.exists: if target_dataset.exists:
for target_snapshot in target_dataset.snapshots: for target_snapshot in target_dataset.snapshots:
if (target_snapshot in target_obsoletes) \ if (target_snapshot in target_obsoletes) \
and (not source_common_snapshot or (target_snapshot.snapshot_name != source_common_snapshot.snapshot_name)): and (not source_common_snapshot or (
target_snapshot.snapshot_name != source_common_snapshot.snapshot_name)):
if target_snapshot.exists: if target_snapshot.exists:
target_snapshot.destroy() target_snapshot.destroy()
@ -1089,36 +1075,40 @@ class ZfsDataset:
# start with snapshots that already exist, minus imcompatibles # start with snapshots that already exist, minus imcompatibles
if target_dataset.exists: if target_dataset.exists:
possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots if snapshot not in incompatible_target_snapshots] possible_target_snapshots = [snapshot for snapshot in target_dataset.snapshots if
snapshot not in incompatible_target_snapshots]
else: else:
possible_target_snapshots = [] possible_target_snapshots = []
# add all snapshots from the source, starting after the common snapshot if it exists # add all snapshots from the source, starting after the common snapshot if it exists
if source_common_snapshot: if source_common_snapshot:
source_snapshot=self.find_next_snapshot(source_common_snapshot ) source_snapshot = self.find_next_snapshot(source_common_snapshot)
else: else:
if self.snapshots: if self.snapshots:
source_snapshot=self.snapshots[0] source_snapshot = self.snapshots[0]
else: else:
source_snapshot=None source_snapshot = None
while source_snapshot: while source_snapshot:
# we want it? # we want it?
if (also_other_snapshots or source_snapshot.is_ours()) and not source_snapshot.is_excluded: if (also_other_snapshots or source_snapshot.is_ours()) and not source_snapshot.is_excluded:
# create virtual target snapshot # create virtual target snapshot
target_snapshot=target_dataset.zfs_node.get_dataset(target_dataset.filesystem_name + "@" + source_snapshot.snapshot_name, force_exists=False) target_snapshot = target_dataset.zfs_node.get_dataset(
target_dataset.filesystem_name + "@" + source_snapshot.snapshot_name, force_exists=False)
possible_target_snapshots.append(target_snapshot) possible_target_snapshots.append(target_snapshot)
source_snapshot = self.find_next_snapshot(source_snapshot) source_snapshot = self.find_next_snapshot(source_snapshot)
### 3: Let the thinner decide what it wants by looking at all the possible target_snaphots at once ### 3: Let the thinner decide what it wants by looking at all the possible target_snaphots at once
if possible_target_snapshots: if possible_target_snapshots:
(target_keeps, target_obsoletes)=target_dataset.zfs_node.thin_list(possible_target_snapshots, keep_snapshots=[possible_target_snapshots[-1]]) (target_keeps, target_obsoletes) = target_dataset.zfs_node.thin_list(possible_target_snapshots,
keep_snapshots=[
possible_target_snapshots[-1]])
else: else:
target_keeps = [] target_keeps = []
target_obsoletes = [] target_obsoletes = []
### 4: Look at what the thinner wants and create a list of snapshots we still need to transfer ### 4: Look at what the thinner wants and create a list of snapshots we still need to transfer
target_transfers=[] target_transfers = []
for target_keep in target_keeps: for target_keep in target_keeps:
if not target_keep.exists: if not target_keep.exists:
target_transfers.append(target_keep) target_transfers.append(target_keep)
@ -1203,7 +1193,7 @@ class ZfsDataset:
target_dataset.handle_incompatible_snapshots(incompatible_target_snapshots, destroy_incompatible) target_dataset.handle_incompatible_snapshots(incompatible_target_snapshots, destroy_incompatible)
# now actually transfer the snapshots, if we want # now actually transfer the snapshots, if we want
if no_send or len(target_transfers)==0: if no_send or len(target_transfers) == 0:
return return
# check if we can resume # check if we can resume
@ -1221,11 +1211,11 @@ class ZfsDataset:
# now actually transfer the snapshots # now actually transfer the snapshots
do_rollback = rollback do_rollback = rollback
prev_source_snapshot=source_common_snapshot prev_source_snapshot = source_common_snapshot
prev_target_snapshot=target_dataset.find_snapshot(source_common_snapshot) prev_target_snapshot = target_dataset.find_snapshot(source_common_snapshot)
for target_snapshot in target_transfers: for target_snapshot in target_transfers:
source_snapshot=self.find_snapshot(target_snapshot) source_snapshot = self.find_snapshot(target_snapshot)
# do the rollback, one time at first transfer # do the rollback, one time at first transfer
if do_rollback: if do_rollback:
@ -1312,8 +1302,8 @@ class ZfsDataset:
self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) self.zfs_node.run(cmd=cmd, valid_exitcodes=[0])
#invalidate cache # invalidate cache
self.__properties=None self.__properties = None
def inherit(self, prop): def inherit(self, prop):
"""inherit zfs property""" """inherit zfs property"""
@ -1326,5 +1316,5 @@ class ZfsDataset:
self.zfs_node.run(cmd=cmd, valid_exitcodes=[0]) self.zfs_node.run(cmd=cmd, valid_exitcodes=[0])
#invalidate cache # invalidate cache
self.__properties=None self.__properties = None

37
zfs_autobackup/compressors.py
View File

@ -7,69 +7,72 @@
COMPRESS_CMDS = { COMPRESS_CMDS = {
'gzip': { 'gzip': {
'cmd': 'gzip', 'cmd': 'gzip',
'args': [ '-3' ], 'args': ['-3'],
'dcmd': 'zcat', 'dcmd': 'zcat',
'dargs': [], 'dargs': [],
}, },
'pigz-fast': { 'pigz-fast': {
'cmd': 'pigz', 'cmd': 'pigz',
'args': [ '-3' ], 'args': ['-3'],
'dcmd': 'pigz', 'dcmd': 'pigz',
'dargs': [ '-dc' ], 'dargs': ['-dc'],
}, },
'pigz-slow': { 'pigz-slow': {
'cmd': 'pigz', 'cmd': 'pigz',
'args': [ '-9' ], 'args': ['-9'],
'dcmd': 'pigz', 'dcmd': 'pigz',
'dargs': [ '-dc' ], 'dargs': ['-dc'],
}, },
'zstd-fast': { 'zstd-fast': {
'cmd': 'zstdmt', 'cmd': 'zstdmt',
'args': [ '-3' ], 'args': ['-3'],
'dcmd': 'zstdmt', 'dcmd': 'zstdmt',
'dargs': [ '-dc' ], 'dargs': ['-dc'],
}, },
'zstd-slow': { 'zstd-slow': {
'cmd': 'zstdmt', 'cmd': 'zstdmt',
'args': [ '-19' ], 'args': ['-19'],
'dcmd': 'zstdmt', 'dcmd': 'zstdmt',
'dargs': [ '-dc' ], 'dargs': ['-dc'],
}, },
'zstd-adapt': { 'zstd-adapt': {
'cmd': 'zstdmt', 'cmd': 'zstdmt',
'args': [ '--adapt' ], 'args': ['--adapt'],
'dcmd': 'zstdmt', 'dcmd': 'zstdmt',
'dargs': [ '-dc' ], 'dargs': ['-dc'],
}, },
'xz': { 'xz': {
'cmd': 'xz', 'cmd': 'xz',
'args': [], 'args': [],
'dcmd': 'xz', 'dcmd': 'xz',
'dargs': [ '-d' ], 'dargs': ['-d'],
}, },
'lzo': { 'lzo': {
'cmd': 'lzop', 'cmd': 'lzop',
'args': [], 'args': [],
'dcmd': 'lzop', 'dcmd': 'lzop',
'dargs': [ '-dfc' ], 'dargs': ['-dfc'],
}, },
'lz4': { 'lz4': {
'cmd': 'lz4', 'cmd': 'lz4',
'args': [], 'args': [],
'dcmd': 'lz4', 'dcmd': 'lz4',
'dargs': [ '-dc' ], 'dargs': ['-dc'],
}, },
} }
def compress_cmd(compressor): def compress_cmd(compressor):
ret=[ COMPRESS_CMDS[compressor]['cmd'] ] ret = [COMPRESS_CMDS[compressor]['cmd']]
ret.extend( COMPRESS_CMDS[compressor]['args']) ret.extend(COMPRESS_CMDS[compressor]['args'])
return ret return ret
def decompress_cmd(compressor): def decompress_cmd(compressor):
ret= [ COMPRESS_CMDS[compressor]['dcmd'] ] ret = [COMPRESS_CMDS[compressor]['dcmd']]
ret.extend(COMPRESS_CMDS[compressor]['dargs']) ret.extend(COMPRESS_CMDS[compressor]['dargs'])
return ret return ret
def choices(): def choices():
return COMPRESS_CMDS.keys() return COMPRESS_CMDS.keys()

26
zfs_autobackup/util.py
View File

@ -1,4 +1,3 @@
# NOTE: surprisingly sha1 in via python3 is faster than the native sha1sum utility, even in the way we use below! # NOTE: surprisingly sha1 in via python3 is faster than the native sha1sum utility, even in the way we use below!
import os import os
import platform import platform
@ -9,19 +8,18 @@ from datetime import datetime
def tmp_name(suffix=""): def tmp_name(suffix=""):
"""create temporary name unique to this process and node. always retruns the same result during the same execution""" """create temporary name unique to this process and node. always retruns the same result during the same execution"""
#we could use uuids but those are ugly and confusing # we could use uuids but those are ugly and confusing
name="{}-{}-{}".format( name = "{}-{}-{}".format(
os.path.basename(sys.argv[0]).replace(" ","_"), os.path.basename(sys.argv[0]).replace(" ", "_"),
platform.node(), platform.node(),
os.getpid()) os.getpid())
name=name+suffix name = name + suffix
return name return name
def get_tmp_clone_name(snapshot): def get_tmp_clone_name(snapshot):
pool=snapshot.zfs_node.get_pool(snapshot) pool = snapshot.zfs_node.get_pool(snapshot)
return pool.name+"/"+tmp_name() return pool.name + "/" + tmp_name()
def output_redir(): def output_redir():
@ -33,10 +31,12 @@ def output_redir():
os.dup2(devnull, sys.stdout.fileno()) os.dup2(devnull, sys.stdout.fileno())
os.dup2(devnull, sys.stderr.fileno()) os.dup2(devnull, sys.stderr.fileno())
def sigpipe_handler(sig, stack): def sigpipe_handler(sig, stack):
#redir output so we dont get more SIGPIPES during cleanup. (which my try to write to stdout) # redir output so we dont get more SIGPIPES during cleanup. (which my try to write to stdout)
output_redir() output_redir()
#deb('redir') # deb('redir')
# def check_output(): # def check_output():
# """make sure stdout still functions. if its broken, this will trigger a SIGPIPE which will be handled by the sigpipe_handler.""" # """make sure stdout still functions. if its broken, this will trigger a SIGPIPE which will be handled by the sigpipe_handler."""
@ -55,9 +55,11 @@ def sigpipe_handler(sig, stack):
# This function will be mocked during unit testing. # This function will be mocked during unit testing.
datetime_now_mock=None datetime_now_mock = None
def datetime_now(utc): def datetime_now(utc):
if datetime_now_mock is None: if datetime_now_mock is None:
return( datetime.utcnow() if utc else datetime.now()) return (datetime.utcnow() if utc else datetime.now())
else: else:
return datetime_now_mock return datetime_now_mock