norohind/zfs_autobackup
1
0
Fork 0
forked from third-party-mirrors/zfs_autobackup
Code Pull Requests Activity

improved progress reporting. improved no_thinning performance

Browse Source
This commit is contained in:
Edwin Eefting 2021-04-23 20:31:37 +02:00
parent 59b0c23a20
commit f0cc2bca2a
4 changed files with 97 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
zfs_autobackup/LogConsole.py
View File

@ -46,3 +46,14 @@ class LogConsole:
else:
print("# " + txt)
sys.stdout.flush()
def progress(self, txt):
"""print progress output to stderr (stays on same line)"""
self.clear_progress()
print(">>> {}\r".format(txt), end='', file=sys.stderr)
sys.stderr.flush()
def clear_progress(self):
import colorama
print(colorama.ansi.clear_line(), end='', file=sys.stderr)
sys.stderr.flush()

49
zfs_autobackup/ZfsAutobackup.py
View File

@ -12,7 +12,7 @@ from zfs_autobackup.ThinnerRule import ThinnerRule
class ZfsAutobackup:
"""main class"""
VERSION = "3.1-beta4"
VERSION = "3.1-beta5"
HEADER = "zfs-autobackup v{} - Copyright 2020 E.H.Eefting (edwin@datux.nl)".format(VERSION)
def __init__(self, argv, print_arguments=True):
@ -149,6 +149,7 @@ class ZfsAutobackup:
self.log.error("Target should not start with a /")
sys.exit(255)
def verbose(self, txt):
self.log.verbose(txt)
@ -162,31 +163,52 @@ class ZfsAutobackup:
self.log.verbose("")
self.log.verbose("#### " + title)
def progress(self, txt):
self.log.progress(txt)
def clear_progress(self):
self.log.clear_progress()
# NOTE: this method also uses self.args. args that need extra processing are passed as function parameters:
def thin_missing_targets(self, target_dataset, used_target_datasets):
"""thin target datasets that are missing on the source."""
self.debug("Thinning obsolete datasets")
missing_datasets=[dataset for dataset in target_dataset.recursive_datasets if dataset not in used_target_datasets]
count=0
for dataset in missing_datasets:
count=count+1
if self.args.progress:
self.progress("Analysing missing {}/{}".format(count, len(missing_datasets)))
for dataset in target_dataset.recursive_datasets:
try:
if dataset not in used_target_datasets:
dataset.debug("Missing on source, thinning")
dataset.thin()
except Exception as e:
dataset.error("Error during thinning of missing datasets ({})".format(str(e)))
if self.args.progress:
self.clear_progress()
# NOTE: this method also uses self.args. args that need extra processing are passed as function parameters:
def destroy_missing_targets(self, target_dataset, used_target_datasets):
"""destroy target datasets that are missing on the source and that meet the requirements"""
self.debug("Destroying obsolete datasets")
for dataset in target_dataset.recursive_datasets:
try:
if dataset not in used_target_datasets:
missing_datasets=[dataset for dataset in target_dataset.recursive_datasets if dataset not in used_target_datasets]
count=0
for dataset in missing_datasets:
count=count+1
if self.args.progress:
self.progress("Analysing destroy missing {}/{}".format(count, len(missing_datasets)))
try:
# cant do anything without our own snapshots
if not dataset.our_snapshots:
if dataset.datasets:
@ -229,6 +251,9 @@ class ZfsAutobackup:
except Exception as e:
dataset.error("Error during --destroy-missing: {}".format(str(e)))
if self.args.progress:
self.clear_progress()
# NOTE: this method also uses self.args. args that need extra processing are passed as function parameters:
def sync_datasets(self, source_node, source_datasets, target_node):
"""Sync datasets, or thin-only on both sides
@ -238,9 +263,15 @@ class ZfsAutobackup:
"""
fail_count = 0
count =0
target_datasets = []
for source_dataset in source_datasets:
# stats
if self.args.progress:
count=count+1
self.progress("Analysing dataset {}/{} ({} failed)".format(count, len(source_datasets), fail_count))
try:
# determine corresponding target_dataset
target_name = self.args.target_path + "/" + source_dataset.lstrip_path(self.args.strip_path)
@ -275,7 +306,11 @@ class ZfsAutobackup:
if self.args.debug:
raise
if self.args.progress:
self.clear_progress()
target_path_dataset = ZfsDataset(target_node, self.args.target_path)
if not self.args.no_thinning:
self.thin_missing_targets(target_dataset=target_path_dataset, used_target_datasets=target_datasets)
if self.args.destroy_missing is not None:
@ -285,7 +320,6 @@ class ZfsAutobackup:
def thin_source(self, source_datasets):
if not self.args.no_thinning:
self.set_title("Thinning source")
for source_dataset in source_datasets:
@ -402,6 +436,7 @@ class ZfsAutobackup:
#no target specified, run in snapshot-only mode
else:
if not self.args.no_thinning:
self.thin_source(source_datasets)
fail_count = 0

1
zfs_autobackup/ZfsDataset.py
View File

@ -564,7 +564,6 @@ class ZfsDataset:
return self.zfs_node.run(cmd, pipe=True, readonly=True)
def recv_pipe(self, pipe, features, filter_properties=None, set_properties=None, ignore_exit_code=False):
"""starts a zfs recv for this snapshot and uses pipe as input

5
zfs_autobackup/ZfsNode.py
View File

@ -128,9 +128,8 @@ class ZfsNode(ExecuteNode):
bytes_left = self._progress_total_bytes - bytes_
minutes_left = int((bytes_left / (bytes_ / (time.time() - self._progress_start_time))) / 60)
print(">>> {}% {}MB/s (total {}MB, {} minutes left) \r".format(percentage, speed, int(
self._progress_total_bytes / (1024 * 1024)), minutes_left), end='', file=sys.stderr)
sys.stderr.flush()
self.logger.progress("Transfer {}% {}MB/s (total {}MB, {} minutes left)".format(percentage, speed, int(
self._progress_total_bytes / (1024 * 1024)), minutes_left))
return