From 78d7dbab6df55f4549900192ef8d6e92875ac4bc Mon Sep 17 00:00:00 2001 From: oddlama Date: Mon, 4 Jul 2022 22:18:45 +0200 Subject: [PATCH] Implement --utc switch to force UTC time (closes #155) --- tests/test_zfsautobackup.py | 2 +- tests/test_zfsnode.py | 12 ++++++------ zfs_autobackup/CliBase.py | 4 +++- zfs_autobackup/ZfsAuto.py | 1 + zfs_autobackup/ZfsAutobackup.py | 11 +++++++---- zfs_autobackup/ZfsAutoverify.py | 6 ++++-- zfs_autobackup/ZfsCheck.py | 2 +- zfs_autobackup/ZfsDataset.py | 21 ++++++++++++++++++--- zfs_autobackup/ZfsNode.py | 3 ++- 9 files changed, 43 insertions(+), 19 deletions(-) diff --git a/tests/test_zfsautobackup.py b/tests/test_zfsautobackup.py index 0a4d2ef..ebae27e 100644 --- a/tests/test_zfsautobackup.py +++ b/tests/test_zfsautobackup.py @@ -892,7 +892,7 @@ test_target1/test_source2/fs2/sub@test-20101111000003 r = shelltest("zfs snapshot test_source1@test") l=LogConsole(show_verbose=True, show_debug=False, color=False) - n=ZfsNode(snapshot_time_format="bla", hold_name="bla", logger=l) + n=ZfsNode(utc=False, snapshot_time_format="bla", hold_name="bla", logger=l) d=ZfsDataset(n,"test_source1@test") sp=d.send_pipe([], prev_snapshot=None, resume_token=None, show_progress=True, raw=False, send_pipes=[], send_properties=True, write_embedded=True, zfs_compressed=True) diff --git a/tests/test_zfsnode.py b/tests/test_zfsnode.py index e916205..b17c51a 100644 --- a/tests/test_zfsnode.py +++ b/tests/test_zfsnode.py @@ -12,7 +12,7 @@ class TestZfsNode(unittest2.TestCase): def test_consistent_snapshot(self): logger = LogStub() description = "[Source]" - node = ZfsNode(snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) with self.subTest("first snapshot"): node.consistent_snapshot(node.selected_datasets(property_name="autobackup:test",exclude_paths=[], exclude_received=False, exclude_unchanged=False, min_change=200000), "test-20101111000001", 100000) @@ -74,7 +74,7 @@ test_target1 def test_consistent_snapshot_prepostcmds(self): logger = LogStub() description = "[Source]" - node = ZfsNode(snapshot_time_format="test", hold_name="test", logger=logger, description=description, debug_output=True) + node = ZfsNode(utc=False, snapshot_time_format="test", hold_name="test", logger=logger, description=description, debug_output=True) with self.subTest("Test if all cmds are executed correctly (no failures)"): with OutputIO() as buf: @@ -137,7 +137,7 @@ test_target1 logger = LogStub() description = "[Source]" - node = ZfsNode(snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) s = pformat(node.selected_datasets(property_name="autobackup:test", exclude_paths=[], exclude_received=False, exclude_unchanged=True, min_change=1)) print(s) @@ -150,7 +150,7 @@ test_target1 def test_validcommand(self): logger = LogStub() description = "[Source]" - node = ZfsNode(snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) with self.subTest("test invalid option"): self.assertFalse(node.valid_command(["zfs", "send", "--invalid-option", "nonexisting"])) @@ -160,7 +160,7 @@ test_target1 def test_supportedsendoptions(self): logger = LogStub() description = "[Source]" - node = ZfsNode(snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description) # -D propably always supported self.assertGreater(len(node.supported_send_options), 0) @@ -168,7 +168,7 @@ test_target1 logger = LogStub() description = "[Source]" # NOTE: this could hang via ssh if we dont close filehandles properly. (which was a previous bug) - node = ZfsNode(snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description, ssh_to='localhost') + node = ZfsNode(utc=False, snapshot_time_format="test-%Y%m%d%H%M%S", hold_name="zfs_autobackup:test", logger=logger, description=description, ssh_to='localhost') self.assertIsInstance(node.supported_recv_options, list) diff --git a/zfs_autobackup/CliBase.py b/zfs_autobackup/CliBase.py index 209ee9f..c8ffe1b 100644 --- a/zfs_autobackup/CliBase.py +++ b/zfs_autobackup/CliBase.py @@ -80,6 +80,8 @@ class CliBase(object): help='show zfs progress output. Enabled automaticly on ttys. (use --no-progress to disable)') group.add_argument('--no-progress', action='store_true', help=argparse.SUPPRESS) # needed to workaround a zfs recv -v bug + group.add_argument('--utc', action='store_true', + help='Use UTC instead of local time when dealing with timestamps for both formatting and parsing. To snapshot in an ISO 8601 compliant time format you may for example specify --snapshot-format "%Y-%m-%dT%H:%M:%SZ". Changing this parameter after-the-fact (existing snapshots) will cause their timestamps to be interpreted as a different time than before.') group.add_argument('--version', action='store_true', help='Show version.') @@ -106,4 +108,4 @@ class CliBase(object): def set_title(self, title): self.log.verbose("") - self.log.verbose("#### " + title) \ No newline at end of file + self.log.verbose("#### " + title) diff --git a/zfs_autobackup/ZfsAuto.py b/zfs_autobackup/ZfsAuto.py index 093428e..660e721 100644 --- a/zfs_autobackup/ZfsAuto.py +++ b/zfs_autobackup/ZfsAuto.py @@ -61,6 +61,7 @@ class ZfsAuto(CliBase): self.verbose("") self.verbose("Selecting dataset property : {}".format(self.property_name)) self.verbose("Snapshot format : {}".format(self.snapshot_time_format)) + self.verbose("Timezone : {}".format("UTC" if args.utc else "Local")) return args diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index f12ff9f..0ce3270 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -1,6 +1,7 @@ import time import argparse +from datetime import datetime from signal import signal, SIGPIPE from .util import output_redir, sigpipe_handler @@ -12,7 +13,6 @@ from .Thinner import Thinner from .ZfsDataset import ZfsDataset from .ZfsNode import ZfsNode from .ThinnerRule import ThinnerRule -import os.path class ZfsAutobackup(ZfsAuto): """The main zfs-autobackup class. Start here, at run() :)""" @@ -435,7 +435,8 @@ class ZfsAutobackup(ZfsAuto): source_thinner = None else: source_thinner = Thinner(self.args.keep_source) - source_node = ZfsNode(snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name, logger=self, + source_node = ZfsNode(utc=self.args.utc, + snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name, logger=self, ssh_config=self.args.ssh_config, ssh_to=self.args.ssh_source, readonly=self.args.test, debug_output=self.args.debug_output, description=description, thinner=source_thinner) @@ -454,7 +455,8 @@ class ZfsAutobackup(ZfsAuto): ################# snapshotting if not self.args.no_snapshot: self.set_title("Snapshotting") - snapshot_name = time.strftime(self.snapshot_time_format) + dt = datetime.utcnow() if self.args.utc else datetime.now() + snapshot_name = dt.strftime(self.snapshot_time_format) source_node.consistent_snapshot(source_datasets, snapshot_name, min_changed_bytes=self.args.min_change, pre_snapshot_cmds=self.args.pre_snapshot_cmd, @@ -471,7 +473,8 @@ class ZfsAutobackup(ZfsAuto): target_thinner = None else: target_thinner = Thinner(self.args.keep_target) - target_node = ZfsNode(snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name, + target_node = ZfsNode(utc=self.args.utc, + snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name, logger=self, ssh_config=self.args.ssh_config, ssh_to=self.args.ssh_target, readonly=self.args.test, debug_output=self.args.debug_output, diff --git a/zfs_autobackup/ZfsAutoverify.py b/zfs_autobackup/ZfsAutoverify.py index dece15a..7568f8f 100644 --- a/zfs_autobackup/ZfsAutoverify.py +++ b/zfs_autobackup/ZfsAutoverify.py @@ -231,7 +231,8 @@ class ZfsAutoverify(ZfsAuto): self.set_title("Source settings") description = "[Source]" - source_node = ZfsNode(snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name, logger=self, + source_node = ZfsNode(utc=self.args.utc, + snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name, logger=self, ssh_config=self.args.ssh_config, ssh_to=self.args.ssh_source, readonly=self.args.test, debug_output=self.args.debug_output, description=description) @@ -249,7 +250,8 @@ class ZfsAutoverify(ZfsAuto): # create target_node self.set_title("Target settings") - target_node = ZfsNode(snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name, + target_node = ZfsNode(utc=self.args.utc, + snapshot_time_format=self.snapshot_time_format, hold_name=self.hold_name, logger=self, ssh_config=self.args.ssh_config, ssh_to=self.args.ssh_target, readonly=self.args.test, debug_output=self.args.debug_output, diff --git a/zfs_autobackup/ZfsCheck.py b/zfs_autobackup/ZfsCheck.py index eae7b9a..190a809 100644 --- a/zfs_autobackup/ZfsCheck.py +++ b/zfs_autobackup/ZfsCheck.py @@ -18,7 +18,7 @@ class ZfsCheck(CliBase): # NOTE: common options argument parsing are in CliBase super(ZfsCheck, self).__init__(argv, print_arguments) - self.node = ZfsNode(self.log, readonly=self.args.test, debug_output=self.args.debug_output) + self.node = ZfsNode(self.log, utc=self.args.utc, readonly=self.args.test, debug_output=self.args.debug_output) self.block_hasher = BlockHasher(count=self.args.count, bs=self.args.block_size, skip=self.args.skip) diff --git a/zfs_autobackup/ZfsDataset.py b/zfs_autobackup/ZfsDataset.py index 7b1c11c..5efca43 100644 --- a/zfs_autobackup/ZfsDataset.py +++ b/zfs_autobackup/ZfsDataset.py @@ -1,4 +1,6 @@ import re +from datetime import datetime +import sys import time from .CachedProperty import CachedProperty @@ -375,9 +377,22 @@ class ZfsDataset: """get timestamp from snapshot name. Only works for our own snapshots with the correct format. """ - - time_secs = time.mktime(time.strptime(self.snapshot_name, self.zfs_node.snapshot_time_format)) - return time_secs + dt = datetime.strptime(self.snapshot_name, self.zfs_node.snapshot_time_format) + if sys.version_info[0] >= 3: + from datetime import timezone + if self.zfs_node.utc: + dt = dt.replace(tzinfo=timezone.utc) + seconds = dt.timestamp() + else: + # python2 has no good functions to deal with UTC. Yet the unix timestamp + # must be in UTC to allow comparison against `time.time()` in on other parts + # of this project (e.g. Thinner.py). If we are handling UTC timestamps, + # we must adjust for that here. + if self.zfs_node.utc: + seconds = (dt - datetime(1970, 1, 1)).total_seconds() + else: + seconds = time.mktime(dt.timetuple()) + return seconds def from_names(self, names): """convert a list of names to a list ZfsDatasets for this zfs_node diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index a230f1c..c635ac4 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -17,10 +17,11 @@ from .ExecuteNode import ExecuteError class ZfsNode(ExecuteNode): """a node that contains zfs datasets. implements global (systemwide/pool wide) zfs commands""" - def __init__(self, logger, snapshot_time_format="", hold_name="", ssh_config=None, ssh_to=None, readonly=False, + def __init__(self, logger, utc=False, snapshot_time_format="", hold_name="", ssh_config=None, ssh_to=None, readonly=False, description="", debug_output=False, thinner=None): + self.utc = utc self.snapshot_time_format = snapshot_time_format self.hold_name = hold_name