third-party-mirrors/zfs_autobackup
1
0
Fork 1
mirror of https://github.com/psy0rz/zfs_autobackup.git synced 2025-06-01 01:20:58 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix resuming on smartos/illumos (actually work around zfs send stderr bug)

Browse Source
This commit is contained in:
Edwin Eefting 2019-11-10 22:18:20 +01:00
parent 8d3c58572b
commit 1ad93da8cc
1 changed files with 23 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
zfs_autobackup
View File

@ -288,8 +288,6 @@ class cached_property(object):
propname=self.func.__name__ propname=self.func.__name__
#store directly in dict so its cached from now on
# value = obj.__dict__[propname] = self.func(obj)
if not hasattr(obj, '_cached_properties'): if not hasattr(obj, '_cached_properties'):
obj._cached_properties={} obj._cached_properties={}
@ -343,12 +341,13 @@ class ExecuteNode:
self.error("STDERR|> "+line.rstrip()) self.error("STDERR|> "+line.rstrip())
def run(self, cmd, input=None, tab_split=False, valid_exitcodes=[ 0 ], readonly=False, hide_errors=False, pipe=False): def run(self, cmd, input=None, tab_split=False, valid_exitcodes=[ 0 ], readonly=False, hide_errors=False, pipe=False, return_stderr=False):
"""run a command on the node """run a command on the node
readonly: make this True if the command doesnt make any changes and is safe to execute in testmode readonly: make this True if the command doesnt make any changes and is safe to execute in testmode
pipe: Instead of executing, return a pipe-handle to be used to input to another run() command. (just like a | in linux) pipe: Instead of executing, return a pipe-handle to be used to input to another run() command. (just like a | in linux)
input: Can be None, a string or a pipe-handle you got from another run() input: Can be None, a string or a pipe-handle you got from another run()
return_stderr: return both stdout and stderr as a tuple
""" """
encoded_cmd=[] encoded_cmd=[]
@ -417,12 +416,16 @@ class ExecuteNode:
selectors=[p.stdout, p.stderr ] selectors=[p.stdout, p.stderr ]
output_lines=[] output_lines=[]
error_lines=[]
while True: while True:
(read_ready, write_ready, ex_ready)=select.select(selectors, [], []) (read_ready, write_ready, ex_ready)=select.select(selectors, [], [])
eof_count=0 eof_count=0
if p.stdout in read_ready: if p.stdout in read_ready:
line=p.stdout.readline().decode('utf-8') line=p.stdout.readline().decode('utf-8')
if line!="": if line!="":
if tab_split:
output_lines.append(line.rstrip().split('\t'))
else:
output_lines.append(line.rstrip()) output_lines.append(line.rstrip())
self._parse_stdout(line) self._parse_stdout(line)
else: else:
@ -430,6 +433,10 @@ class ExecuteNode:
if p.stderr in read_ready: if p.stderr in read_ready:
line=p.stderr.readline().decode('utf-8') line=p.stderr.readline().decode('utf-8')
if line!="": if line!="":
if tab_split:
error_lines.append(line.rstrip().split('\t'))
else:
error_lines.append(line.rstrip())
self._parse_stderr(line, hide_errors) self._parse_stderr(line, hide_errors)
else: else:
eof_count=eof_count+1 eof_count=eof_count+1
@ -460,13 +467,10 @@ class ExecuteNode:
if valid_exitcodes and p.returncode not in valid_exitcodes: if valid_exitcodes and p.returncode not in valid_exitcodes:
raise(subprocess.CalledProcessError(p.returncode, encoded_cmd)) raise(subprocess.CalledProcessError(p.returncode, encoded_cmd))
if not tab_split: if return_stderr:
return(output_lines) return ( output_lines, error_lines )
else: else:
ret=[] return(output_lines)
for line in output_lines:
ret.append(line.split("\t"))
return(ret)
@ -479,7 +483,7 @@ class ZfsDataset():
""" """
# illegal properties per dataset type. these will be filtered from --set-properties and --filter-properties # illegal properties per dataset type. these will be removed from --set-properties and --filter-properties
ILLEGAL_PROPERTIES={ ILLEGAL_PROPERTIES={
'filesystem': [ ], 'filesystem': [ ],
'volume': [ "canmount" ], 'volume': [ "canmount" ],
@ -933,9 +937,13 @@ class ZfsDataset():
def get_resume_snapshot(self, resume_token): def get_resume_snapshot(self, resume_token):
"""returns snapshot that will be resumed by this resume token (run this on source with target-token)""" """returns snapshot that will be resumed by this resume token (run this on source with target-token)"""
#use zfs send -n option to determine this #use zfs send -n option to determine this
lines=self.zfs_node.run([ "zfs", "send", "-t", resume_token, "-n", "-v" ], valid_exitcodes=[ 0, 255 ], readonly=True) #NOTE: on smartos stderr, on linux stdout
( stdout, stderr )=self.zfs_node.run([ "zfs", "send", "-t", resume_token, "-n", "-v" ], valid_exitcodes=[ 0, 255 ], readonly=True, return_stderr=True )
if stdout:
lines=stdout
else:
lines=stderr
for line in lines: for line in lines:
matches=re.findall("toname = .*@(.*)", line) matches=re.findall("toname = .*@(.*)", line)
if matches: if matches:
@ -947,22 +955,6 @@ class ZfsDataset():
return(None) return(None)
# def resume_transfer(self, target_dataset, show_progress=False, filter_properties=[], set_properties=[], ignore_recv_exit_code=False):
# """resume an interrupted transfer, if there is one"""
#
# #resume is a kind of special case since we dont know which snapshot we are transferring. (its encoded in the resume token)
# if 'receive_resume_token' in target_dataset.properties:
# target_dataset.verbose("resuming")
# snapshot=self.get_resume_snapshot(target_dataset.properties['receive_resume_token'])
# p(snapshot)
# sys.exit(1)
#
# #just send and recv on dataset instead of snapshot object.
# pipe=self.send_pipe(show_progress=show_progress, resume_token=target_dataset.properties['receive_resume_token'])
# target_dataset.recv_pipe(pipe,resume=True, filter_properties=filter_properties, set_properties=set_properties, ignore_exit_code=ignore_recv_exit_code)
# return(True)
#
# return(False)
def thin(self, keeps=[]): def thin(self, keeps=[]):
@ -1286,7 +1278,7 @@ class ZfsAutobackup:
def __init__(self): def __init__(self):
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description='ZFS autobackup v3.0-beta1', description='ZFS autobackup v3.0-beta2',
epilog='When a filesystem fails, zfs_backup will continue and report the number of failures at that end. Also the exit code will indicate the number of failures.') epilog='When a filesystem fails, zfs_backup will continue and report the number of failures at that end. Also the exit code will indicate the number of failures.')
parser.add_argument('--ssh-source', default=None, help='Source host to get backup from. (user@hostname) Default %(default)s.') parser.add_argument('--ssh-source', default=None, help='Source host to get backup from. (user@hostname) Default %(default)s.')
parser.add_argument('--ssh-target', default=None, help='Target host to push backup to. (user@hostname) Default %(default)s.') parser.add_argument('--ssh-target', default=None, help='Target host to push backup to. (user@hostname) Default %(default)s.')
@ -1321,7 +1313,7 @@ class ZfsAutobackup:
parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)') parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)')
parser.add_argument('--verbose', action='store_true', help='verbose output') parser.add_argument('--verbose', action='store_true', help='verbose output')
parser.add_argument('--debug', action='store_true', help='Show zfs commands that are executed.') parser.add_argument('--debug', action='store_true', help='Show zfs commands that are executed, stops after an exception.')
parser.add_argument('--debug-output', action='store_true', help='Show zfs commands and their output/exit codes. (noisy)') parser.add_argument('--debug-output', action='store_true', help='Show zfs commands and their output/exit codes. (noisy)')
parser.add_argument('--progress', action='store_true', help='show zfs progress output (to stderr)') parser.add_argument('--progress', action='store_true', help='show zfs progress output (to stderr)')