third-party-mirrors/zfs_autobackup
1
0
Fork 1
mirror of https://github.com/psy0rz/zfs_autobackup.git synced 2025-04-13 22:47:12 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Edwin Eefting 2019-10-21 13:08:50 +02:00
parent 04cc860db3
commit 058a189aa5
1 changed files with 50 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
zfs_autobackup
View File

@ -116,13 +116,15 @@ class ExecuteNode:
"""an endpoint to execute local or remote commands via ssh""" """an endpoint to execute local or remote commands via ssh"""
def __init__(self, ssh_to=None, readonly=False): def __init__(self, ssh_to=None, readonly=False, debug_output=False):
"""ssh_to: server you want to ssh to. none means local """ssh_to: server you want to ssh to. none means local
readonly: only execute commands that dont make any changes (usefull for testing-runs) readonly: only execute commands that dont make any changes (usefull for testing-runs)
debug_output: show output and exit codes of commands in debugging output.
""" """
self.ssh_to=ssh_to self.ssh_to=ssh_to
self.readonly=readonly self.readonly=readonly
self.debug_output=debug_output
def run(self, cmd, input=None, tab_split=False, valid_exitcodes=[ 0 ], readonly=False, hide_errors=False, pipe=False): def run(self, cmd, input=None, tab_split=False, valid_exitcodes=[ 0 ], readonly=False, hide_errors=False, pipe=False):
@ -202,7 +204,8 @@ class ExecuteNode:
line=p.stdout.readline() line=p.stdout.readline()
if line!="": if line!="":
output_lines.append(line.rstrip()) output_lines.append(line.rstrip())
self.debug("STDOUT > "+line.rstrip()) if self.debug_output:
self.debug("STDOUT > "+line.rstrip())
else: else:
eof_count=eof_count+1 eof_count=eof_count+1
if p.stderr in read_ready: if p.stderr in read_ready:
@ -228,34 +231,20 @@ class ExecuteNode:
if p.poll()!=None and ((not isinstance(input, subprocess.Popen)) or input.poll()!=None) and eof_count==len(selectors): if p.poll()!=None and ((not isinstance(input, subprocess.Popen)) or input.poll()!=None) and eof_count==len(selectors):
break break
# if isinstance(input, subprocess.Popen):
# (output, errors)=p.communicate()
# else:
# (output, errors)=p.communicate(input=input)
#handle piped process error output and exit codes #handle piped process error output and exit codes
if isinstance(input, subprocess.Popen): if isinstance(input, subprocess.Popen):
# pipe_outputs=input.communicate()
# if not hide_errors:
# for line in pipe_outputs[1].splitlines():
# self.error("Pipe-error: "+line)
self.debug("EXIT |> {}".format(input.returncode)) if self.debug_output:
self.debug("EXIT |> {}".format(input.returncode))
if input.returncode not in valid_exitcodes: if input.returncode not in valid_exitcodes:
raise(subprocess.CalledProcessError(input.returncode, "(pipe)")) raise(subprocess.CalledProcessError(input.returncode, "(pipe)"))
#handle error output and exit codes
# if not hide_errors:
# for line in errors.splitlines():
# self.error(line)
self.debug("EXIT > {}".format(p.returncode)) if self.debug_output:
self.debug("EXIT > {}".format(p.returncode))
if p.returncode not in valid_exitcodes: if p.returncode not in valid_exitcodes:
raise(subprocess.CalledProcessError(p.returncode, encoded_cmd)) raise(subprocess.CalledProcessError(p.returncode, encoded_cmd))
# lines=output.splitlines()
if not tab_split: if not tab_split:
return(output_lines) return(output_lines)
else: else:
@ -375,7 +364,7 @@ class ZfsDataset():
names=self.zfs_node.run(cmd=cmd, readonly=True) names=self.zfs_node.run(cmd=cmd, readonly=True)
return(self.from_names(names)) return(self.from_names(names))
@cached_property @property
def our_snapshots(self): def our_snapshots(self):
"""get list of snapshots creates by us of this dataset""" """get list of snapshots creates by us of this dataset"""
self.debug("Getting our snapshots") self.debug("Getting our snapshots")
@ -527,50 +516,47 @@ class ZfsDataset():
connects a send_pipe() to recv_pipe() connects a send_pipe() to recv_pipe()
""" """
self.debug("Transfer snapshot") self.debug("Transfer snapshot to {}".format(target_dataset))
#initial or resume
if not prev_snapshot or resume_token:
if resume_token:
target_dataset.verbose("receiving resumed") #we dont know which one XXX: fix, we need to know for testmode
else:
target_dataset.verbose("receiving @{}".format(self.snapshot_name))
pipe=self.send_pipe(resume=resume, resume_token=resume_token, show_progress=show_progress)
else:
#incemental
target_dataset.verbose("receiving @{}...@{}".format(prev_snapshot.snapshot_name, self.snapshot_name))
pipe=self.send_pipe(resume=resume, show_progress=show_progress, prev_snapshot=prev_snapshot)
#do it
target_dataset.recv_pipe(pipe)
if resume_token: if resume_token:
#we dont know which snapshot it was, so invalidate cache target_dataset.verbose("resuming")
target_dataset.invalidate()
#initial or resume
if not prev_snapshot:
target_dataset.verbose("receiving @{} (new)".format(self.snapshot_name))
else: else:
#update cache #incemental
target_dataset.snapshots.append(ZfsDataset(target_dataset.zfs_node, target_dataset.name+"@"+self.snapshot_name)) target_dataset.verbose("receiving @{}".format(self.snapshot_name))
#do it
pipe=self.send_pipe(resume=resume, show_progress=show_progress, resume_token=resume_token, prev_snapshot=prev_snapshot)
target_dataset.recv_pipe(pipe)
#update cache
target_dataset.snapshots.append(ZfsDataset(target_dataset.zfs_node, target_dataset.name+"@"+self.snapshot_name))
def sync_snapshots(self, target_dataset, show_progress=False): def sync_snapshots(self, target_dataset, show_progress=False):
"""sync our snapshots to target_dataset""" """sync our snapshots to target_dataset"""
self.debug("Sync snapshots")
# inital transfer # inital transfer
resume_token=None
if not target_dataset.exists: if not target_dataset.exists:
self.debug("Initial transfer") self.debug("Sync snapshots: Initial transfer")
self.our_snapshots[0].transfer_snapshot(target_dataset, show_progress=True) self.our_snapshots[0].transfer_snapshot(target_dataset, show_progress=show_progress)
else: else:
#on resuming we dont need to know anything, the token is enough #filesystem exists, need to resume something?
if 'receive_resume_token' in target_dataset.properties: if 'receive_resume_token' in target_dataset.properties:
self.debug("Resume transfer") self.debug("Sync snapshots: Found resume token")
self.transfer_snapshot(target_dataset, resume_token=target_dataset.properties['receive_resume_token'], show_progress=show_progress) resume_token=target_dataset.properties['receive_resume_token']
self.debug("Incremental transfer") #resume initial transfer?
if len(target_dataset.our_snapshots)==0:
self.debug("Sync snapshots: Resuming inital transfer")
self.our_snapshots[0].transfer_snapshot(target_dataset, show_progress=show_progress, resume_token=resume_token)
resume_token=None
self.debug("Sync snapshots: Incremental transfer")
latest_common_snapshot=None latest_common_snapshot=None
for source_snapshot in self.our_snapshots: for source_snapshot in self.our_snapshots:
target_snapshot=target_dataset.find_snapshot(source_snapshot.snapshot_name) target_snapshot=target_dataset.find_snapshot(source_snapshot.snapshot_name)
@ -580,7 +566,8 @@ class ZfsDataset():
else: else:
if latest_common_snapshot: if latest_common_snapshot:
#transfer it #transfer it
source_snapshot.transfer_snapshot(target_dataset, latest_common_snapshot, show_progress=True) source_snapshot.transfer_snapshot(target_dataset, latest_common_snapshot, show_progress=True, resume_token=resume_token)
resume_token=None
latest_common_snapshot=source_snapshot latest_common_snapshot=source_snapshot
if not latest_common_snapshot: if not latest_common_snapshot:
@ -591,7 +578,7 @@ class ZfsDataset():
class ZfsNode(ExecuteNode): class ZfsNode(ExecuteNode):
"""a node that contains zfs datasets. implements global lowlevel zfs commands""" """a node that contains zfs datasets. implements global lowlevel zfs commands"""
def __init__(self, backup_name, zfs_autobackup, ssh_to=None, readonly=False, description=""): def __init__(self, backup_name, zfs_autobackup, ssh_to=None, readonly=False, description="", debug_output=False):
self.backup_name=backup_name self.backup_name=backup_name
if not description: if not description:
self.description=ssh_to self.description=ssh_to
@ -600,7 +587,7 @@ class ZfsNode(ExecuteNode):
self.zfs_autobackup=zfs_autobackup #for logging self.zfs_autobackup=zfs_autobackup #for logging
ExecuteNode.__init__(self, ssh_to=ssh_to, readonly=readonly) ExecuteNode.__init__(self, ssh_to=ssh_to, readonly=readonly, debug_output=debug_output)
def verbose(self,txt): def verbose(self,txt):
self.zfs_autobackup.verbose("{} {}".format(self.description, txt)) self.zfs_autobackup.verbose("{} {}".format(self.description, txt))
@ -726,13 +713,18 @@ class ZfsAutobackup:
parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)') parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)')
parser.add_argument('--verbose', action='store_true', help='verbose output') parser.add_argument('--verbose', action='store_true', help='verbose output')
parser.add_argument('--debug', action='store_true', help='debug output (shows commands that are executed)') parser.add_argument('--debug', action='store_true', help='Show zfs commands that are executed.')
parser.add_argument('--debug-output', action='store_true', help='Show zfs commands and their output/exit codes. (noisy)')
parser.add_argument('--progress', action='store_true', help='show zfs progress output (to stderr)') parser.add_argument('--progress', action='store_true', help='show zfs progress output (to stderr)')
#note args is the only global variable we use, since its a global readonly setting anyway #note args is the only global variable we use, since its a global readonly setting anyway
args = parser.parse_args() args = parser.parse_args()
self.args=args self.args=args
if args.debug_output:
args.debug=True
self.log=Log(show_debug=self.args.debug, show_verbose=self.args.verbose) self.log=Log(show_debug=self.args.debug, show_verbose=self.args.verbose)
@ -751,10 +743,10 @@ class ZfsAutobackup:
def run(self): def run(self):
description="[Source]" description="[Source]"
source_node=ZfsNode(self.args.backup_name, self, ssh_to=self.args.ssh_source, readonly=self.args.test, description=description) source_node=ZfsNode(self.args.backup_name, self, ssh_to=self.args.ssh_source, readonly=self.args.test, debug_output=self.args.debug_output, description=description)
description="[Target]" description="[Target]"
target_node=ZfsNode(self.args.backup_name, self, ssh_to=self.args.ssh_target, readonly=self.args.test, description=description) target_node=ZfsNode(self.args.backup_name, self, ssh_to=self.args.ssh_target, readonly=self.args.test, debug_output=self.args.debug_output, description=description)
# target_node.run(["/root/outputtest"], readonly=True) # target_node.run(["/root/outputtest"], readonly=True)
self.set_title("Selecting") self.set_title("Selecting")