From 5cca819916bd12fa6b3c3deb74bbd724b92823d7 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Mon, 2 Oct 2023 16:29:46 +0200 Subject: [PATCH] central timehandling and better mocking during test --- tests/basetest.py | 20 +++++ tests/test_zfsautobackup32.py | 48 ++++++++---- zfs_autobackup/Thinner.py | 8 +- zfs_autobackup/ZfsAuto.py | 4 +- zfs_autobackup/ZfsAutobackup.py | 9 +-- zfs_autobackup/ZfsNode.py | 4 +- zfs_autobackup/test.py | 129 -------------------------------- zfs_autobackup/util.py | 26 +++---- 8 files changed, 75 insertions(+), 173 deletions(-) delete mode 100644 zfs_autobackup/test.py diff --git a/tests/basetest.py b/tests/basetest.py index 3994b81..18c15fd 100644 --- a/tests/basetest.py +++ b/tests/basetest.py @@ -4,6 +4,8 @@ import sys +import zfs_autobackup.util + #dirty hack for this error: #AttributeError: module 'collections' has no attribute 'MutableMapping' @@ -28,6 +30,9 @@ import contextlib import sys import io +import datetime + + TEST_POOLS="test_source1 test_source2 test_target1" ZFS_USERSPACE= subprocess.check_output("dpkg-query -W zfsutils-linux |cut -f2", shell=True).decode('utf-8').rstrip() ZFS_KERNEL= subprocess.check_output("modinfo zfs|grep ^version |sed 's/.* //'", shell=True).decode('utf-8').rstrip() @@ -105,3 +110,18 @@ def prepare_zpools(): subprocess.check_call("zfs set autobackup:test=child test_source2/fs2", shell=True) print("Prepare done") + + + +@contextlib.contextmanager +def mocktime(time_str, format="%Y%m%d%H%M%S"): + + def fake_datetime_now(): + return datetime.datetime.strptime(time_str, format) + + with patch.object(zfs_autobackup.util,'datetime_now_mock', fake_datetime_now()): + yield + + + + diff --git a/tests/test_zfsautobackup32.py b/tests/test_zfsautobackup32.py index abd60dd..9b59923 100644 --- a/tests/test_zfsautobackup32.py +++ b/tests/test_zfsautobackup32.py @@ -1,5 +1,4 @@ from basetest import * -import time class TestZfsAutobackup32(unittest2.TestCase): """various new 3.2 features""" @@ -137,18 +136,39 @@ test_target1/test_source2/fs2/sub@test-20101111000001 """) - # def test_stuff(self): - # - # - # shelltest("zfs set autobackup:test=true test_source2") - # # shelltest("zfs set readonly=on test_target1") - # - # with patch('time.strftime', return_value="test-20101111000000"): - # self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --clear-mountpoint".split(" ")).run()) - # - # # shelltest("zfs mount test_target1/test_source2/fs2/sub" ) - # - # with patch('time.strftime', return_value="test-20101111000001"): - # self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --rollback".split(" ")).run()) + #XXX: VERBERTERING VAN ADD VIRTUALSNAPSHOTS IN GIT STASH! + def test_thinning(self): + + # time_str = "20111112000000" # month in the "future" + # future_timestamp = time_secs = time.mktime(time.strptime(time_str, "%Y%m%d%H%M%S")) + # with patch('time.time', return_value=future_timestamp): + + with mocktime("20001001000000"): + print(datetime_now(False)) + self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint --verbose".split(" ")).run()) + +# with patch('time.strftime', return_value="test-20001101000000"): +# self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint test_target1 --no-progress --allow-empty --clear-mountpoint".split(" ")).run()) +# +# with patch('time.strftime', return_value="test-20001201000000"): +# self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint".split(" ")).run()) +# +# with patch('time.strftime', return_value="test-20001202000000"): +# self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint".split(" ")).run()) +# +# time_str="test-20001203000000" +# with patch('time.time', return_value=time.mktime(time.strptime(time_str, "test-%Y%m%d%H%M%S"))): +# with patch('time.strftime', return_value=time_str): +# self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --allow-empty --clear-mountpoint --keep-source=1d2d".split(" ")).run()) +# +# +# +# r=shelltest("zfs list -H -o name -r -t snapshot test_source1 test_target1") +# self.assertMultiLineEqual(r,""" +# /test_target1 +# /test_target1/test_source1/fs1 +# /test_target1/test_source1/fs1/sub +# /test_target1/test_source2/fs2/sub +# """) diff --git a/zfs_autobackup/Thinner.py b/zfs_autobackup/Thinner.py index 1f5cfb3..0718505 100644 --- a/zfs_autobackup/Thinner.py +++ b/zfs_autobackup/Thinner.py @@ -1,4 +1,3 @@ -import time from .ThinnerRule import ThinnerRule @@ -37,7 +36,7 @@ class Thinner: return ret - def thin(self, objects, keep_objects=None, now=None): + def thin(self, objects, keep_objects, now): """thin list of objects with current schedule rules. objects: list of objects to thin. every object should have timestamp attribute. @@ -49,8 +48,6 @@ class Thinner: now: if specified, use this time as current time """ - if not keep_objects: - keep_objects = [] # always keep a number of the last objets? if self.always_keep: @@ -68,9 +65,6 @@ class Thinner: for rule in self.rules: time_blocks[rule.period] = {} - if not now: - now = int(time.time()) - keeps = [] removes = [] diff --git a/zfs_autobackup/ZfsAuto.py b/zfs_autobackup/ZfsAuto.py index 6f600ce..39c8a19 100644 --- a/zfs_autobackup/ZfsAuto.py +++ b/zfs_autobackup/ZfsAuto.py @@ -1,8 +1,8 @@ import argparse import sys -from datetime import time, datetime from .CliBase import CliBase +from .util import datetime_now class ZfsAuto(CliBase): @@ -59,7 +59,7 @@ class ZfsAuto(CliBase): self.snapshot_time_format = args.snapshot_format.format(args.backup_name) self.hold_name = args.hold_format.format(args.backup_name) - dt = datetime.utcnow() if args.utc else datetime.now() + dt = datetime_now(args.utc) self.verbose("") self.verbose("Current time {} : {}".format(args.utc and "UTC" or " ", dt.strftime("%Y-%m-%d %H:%M:%S"))) diff --git a/zfs_autobackup/ZfsAutobackup.py b/zfs_autobackup/ZfsAutobackup.py index 7fe6930..6b0b2a3 100644 --- a/zfs_autobackup/ZfsAutobackup.py +++ b/zfs_autobackup/ZfsAutobackup.py @@ -1,9 +1,7 @@ -import time import argparse -from datetime import datetime from signal import signal, SIGPIPE -from .util import output_redir, sigpipe_handler +from .util import output_redir, sigpipe_handler, datetime_now from .ZfsAuto import ZfsAuto @@ -203,7 +201,7 @@ class ZfsAutobackup(ZfsAuto): else: # past the deadline? deadline_ttl = ThinnerRule("0s" + self.args.destroy_missing).ttl - now = int(time.time()) + now = datetime_now(self.args.utc).timestamp() if dataset.our_snapshots[-1].timestamp + deadline_ttl > now: dataset.verbose("Destroy missing: Waiting for deadline.") else: @@ -461,8 +459,7 @@ class ZfsAutobackup(ZfsAuto): ################# snapshotting if not self.args.no_snapshot: self.set_title("Snapshotting") - dt = datetime.utcnow() if self.args.utc else datetime.now() - snapshot_name = dt.strftime(self.snapshot_time_format) + snapshot_name = datetime_now(self.args.utc).strftime(self.snapshot_time_format) source_node.consistent_snapshot(source_datasets, snapshot_name, min_changed_bytes=self.args.min_change, pre_snapshot_cmds=self.args.pre_snapshot_cmd, diff --git a/zfs_autobackup/ZfsNode.py b/zfs_autobackup/ZfsNode.py index b87a118..560aa62 100644 --- a/zfs_autobackup/ZfsNode.py +++ b/zfs_autobackup/ZfsNode.py @@ -12,6 +12,7 @@ from .CachedProperty import CachedProperty from .ZfsPool import ZfsPool from .ZfsDataset import ZfsDataset from .ExecuteNode import ExecuteError +from .util import datetime_now class ZfsNode(ExecuteNode): @@ -59,7 +60,8 @@ class ZfsNode(ExecuteNode): def thin(self, objects, keep_objects): # NOTE: if thinning is disabled with --no-thinning, self.__thinner will be none. if self.__thinner is not None: - return self.__thinner.thin(objects, keep_objects) + + return self.__thinner.thin(objects, keep_objects, datetime_now(self.utc)) else: return (keep_objects, []) diff --git a/zfs_autobackup/test.py b/zfs_autobackup/test.py deleted file mode 100644 index 14b6bba..0000000 --- a/zfs_autobackup/test.py +++ /dev/null @@ -1,129 +0,0 @@ -import os.path -import os -import subprocess -import sys -import time -from signal import signal, SIGPIPE - -import util - -signal(SIGPIPE, util.sigpipe_handler) - - -try: - print ("voor eerste") - raise Exception("eerstre") -except Exception as e: - print ("voor tweede") - raise Exception("tweede") -finally: - print ("JO") - -def generator(): - - try: - util.deb('in generator') - print ("TRIGGER SIGPIPE") - sys.stdout.flush() - util.deb('after trigger') - - # if False: - yield ("bla") - # yield ("bla") - - except GeneratorExit as e: - util.deb('GENEXIT '+str(e)) - raise - - except Exception as e: - util.deb('EXCEPT '+str(e)) - finally: - util.deb('FINALLY') - print("nog iets") - sys.stdout.flush() - util.deb('after print in finally WOOP!') - - -util.deb('START') -g=generator() -util.deb('after generator') -for bla in g: - # print ("heb wat ontvangen") - util.deb('ontvangen van gen') - break - # raise Exception("moi") - - pass -raise Exception("moi") - -util.deb('after for') - -while True: - pass - -# -# with open('test.py', 'rb') as fh: -# -# # fsize = fh.seek(10000, os.SEEK_END) -# # print(fsize) -# -# start=time.time() -# for i in range(0,1000000): -# # fh.seek(0, 0) -# fsize=fh.seek(0, os.SEEK_END) -# # fsize=fh.tell() -# # os.path.getsize('test.py') -# print(time.time()-start) -# -# -# print(fh.tell()) -# -# sys.exit(0) -# -# -# -# checked=1 -# skipped=1 -# coverage=0.1 -# -# max_skip=0 -# -# -# skipinarow=0 -# while True: -# total=checked+skipped -# -# skip=coveragemax_skip: -# max_skip=skipinarow -# else: -# skipinarow=0 -# checked=checked+1 -# print("C {:.2f}%".format(checked * 100 / total)) -# -# print(max_skip) -# -# skip=0 -# while True: -# -# total=checked+skipped -# if skip>0: -# skip=skip-1 -# skipped = skipped + 1 -# print("S {:.2f}%".format(checked * 100 / total)) -# else: -# checked=checked+1 -# print("C {:.2f}%".format(checked * 100 / total)) -# -# #calc new skip -# skip=skip+((1/coverage)-1)*(random()*2) -# # print(skip) -# if skip> max_skip: -# max_skip=skip -# -# print(max_skip) diff --git a/zfs_autobackup/util.py b/zfs_autobackup/util.py index 4d774b3..db6f07c 100644 --- a/zfs_autobackup/util.py +++ b/zfs_autobackup/util.py @@ -1,21 +1,9 @@ -# root@psyt14s:/home/psy/zfs_autobackup# ls -lh /home/psy/Downloads/carimage.zip -# -rw-rw-r-- 1 psy psy 990M Nov 26 2020 /home/psy/Downloads/carimage.zip -# root@psyt14s:/home/psy/zfs_autobackup# time sha1sum /home/psy/Downloads/carimage.zip -# a682e1a36e16fe0d0c2f011104f4a99004f19105 /home/psy/Downloads/carimage.zip -# -# real 0m2.558s -# user 0m2.105s -# sys 0m0.448s -# root@psyt14s:/home/psy/zfs_autobackup# time python3 -m zfs_autobackup.ZfsCheck -# -# real 0m1.459s -# user 0m0.993s -# sys 0m0.462s # NOTE: surprisingly sha1 in via python3 is faster than the native sha1sum utility, even in the way we use below! import os import platform import sys +from datetime import datetime def tmp_name(suffix=""): @@ -48,7 +36,7 @@ def output_redir(): def sigpipe_handler(sig, stack): #redir output so we dont get more SIGPIPES during cleanup. (which my try to write to stdout) output_redir() - deb('redir') + #deb('redir') # def check_output(): # """make sure stdout still functions. if its broken, this will trigger a SIGPIPE which will be handled by the sigpipe_handler.""" @@ -63,3 +51,13 @@ def sigpipe_handler(sig, stack): # fh.write("DEB: "+txt+"\n") +# This should be the only source of trueth for the current datetime. +# This function will be mocked during unit testing. + + +datetime_now_mock=None +def datetime_now(utc): + if datetime_now_mock is None: + return( datetime.utcnow() if utc else datetime.now()) + else: + return datetime_now_mock