central timehandling and better mocking during test

tests/basetest.py

@@ -4,6 +4,8 @@
 
 import sys
 
+import zfs_autobackup.util
+
 #dirty hack for this error:
 #AttributeError: module 'collections' has no attribute 'MutableMapping'
 
@@ -28,6 +30,9 @@ import contextlib
 import sys
 import io
 
+import datetime
+
+
 TEST_POOLS="test_source1 test_source2 test_target1"
 ZFS_USERSPACE=  subprocess.check_output("dpkg-query -W zfsutils-linux |cut -f2", shell=True).decode('utf-8').rstrip()
 ZFS_KERNEL=     subprocess.check_output("modinfo zfs|grep ^version |sed 's/.* //'", shell=True).decode('utf-8').rstrip()
@@ -105,3 +110,18 @@ def prepare_zpools():
     subprocess.check_call("zfs set autobackup:test=child test_source2/fs2", shell=True)
 
     print("Prepare done")
+
+
+
+@contextlib.contextmanager
+def mocktime(time_str, format="%Y%m%d%H%M%S"):
+
+    def fake_datetime_now():
+        return datetime.datetime.strptime(time_str, format)
+
+    with patch.object(zfs_autobackup.util,'datetime_now_mock', fake_datetime_now()):
+        yield
+
+
+
+

tests/test_zfsautobackup32.py

@@ -1,5 +1,4 @@
 from basetest import *
-import time
 
 class TestZfsAutobackup32(unittest2.TestCase):
     """various new 3.2 features"""
@@ -137,18 +136,39 @@ test_target1/test_source2/fs2/sub@test-20101111000001
 """)
 
 
-    # def test_stuff(self):
-    #
-    #
-    #     shelltest("zfs set autobackup:test=true test_source2")
-    #     # shelltest("zfs set readonly=on test_target1")
-    #
-    #     with patch('time.strftime', return_value="test-20101111000000"):
-    #         self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --clear-mountpoint".split(" ")).run())
-    #
-    #     # shelltest("zfs mount test_target1/test_source2/fs2/sub" )
-    #
-    #     with patch('time.strftime', return_value="test-20101111000001"):
-    #         self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --verbose --allow-empty --rollback".split(" ")).run())
 
 
+    #XXX: VERBERTERING VAN ADD VIRTUALSNAPSHOTS IN GIT STASH!
+    def test_thinning(self):
+
+        # time_str = "20111112000000"  # month in the "future"
+        # future_timestamp = time_secs = time.mktime(time.strptime(time_str, "%Y%m%d%H%M%S"))
+        # with patch('time.time', return_value=future_timestamp):
+
+        with  mocktime("20001001000000"):
+            print(datetime_now(False))
+            self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint --verbose".split(" ")).run())
+
+#         with patch('time.strftime', return_value="test-20001101000000"):
+#             self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint test_target1 --no-progress --allow-empty --clear-mountpoint".split(" ")).run())
+#
+#         with patch('time.strftime', return_value="test-20001201000000"):
+#             self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint".split(" ")).run())
+#
+#         with patch('time.strftime', return_value="test-20001202000000"):
+#             self.assertFalse(ZfsAutobackup("test --allow-empty --clear-mountpoint".split(" ")).run())
+#
+#         time_str="test-20001203000000"
+#         with patch('time.time', return_value=time.mktime(time.strptime(time_str, "test-%Y%m%d%H%M%S"))):
+#             with patch('time.strftime', return_value=time_str):
+#                 self.assertFalse(ZfsAutobackup("test test_target1 --no-progress --allow-empty --clear-mountpoint --keep-source=1d2d".split(" ")).run())
+#
+#
+#
+#             r=shelltest("zfs list -H -o name -r -t snapshot test_source1 test_target1")
+#             self.assertMultiLineEqual(r,"""
+# /test_target1
+# /test_target1/test_source1/fs1
+# /test_target1/test_source1/fs1/sub
+# /test_target1/test_source2/fs2/sub
+# """)

zfs_autobackup/Thinner.py

@@ -1,4 +1,3 @@
-import time
 
 from .ThinnerRule import ThinnerRule
 
@@ -37,7 +36,7 @@ class Thinner:
 
         return ret
 
-    def thin(self, objects, keep_objects=None, now=None):
+    def thin(self, objects, keep_objects, now):
         """thin list of objects with current schedule rules. objects: list of
         objects to thin. every object should have timestamp attribute.
 
@@ -49,8 +48,6 @@ class Thinner:
             now: if specified, use this time as current time
         """
 
-        if not keep_objects:
-            keep_objects = []
 
         # always keep a number of the last objets?
         if self.always_keep:
@@ -68,9 +65,6 @@ class Thinner:
         for rule in self.rules:
             time_blocks[rule.period] = {}
 
-        if not now:
-            now = int(time.time())
-
         keeps = []
         removes = []
 

zfs_autobackup/ZfsAuto.py

@@ -1,8 +1,8 @@
 import argparse
 import sys
-from datetime import time, datetime
 
 from .CliBase import CliBase
+from .util import datetime_now
 
 
 class ZfsAuto(CliBase):
@@ -59,7 +59,7 @@ class ZfsAuto(CliBase):
         self.snapshot_time_format = args.snapshot_format.format(args.backup_name)
         self.hold_name = args.hold_format.format(args.backup_name)
 
-        dt = datetime.utcnow() if args.utc else datetime.now()
+        dt = datetime_now(args.utc)
 
         self.verbose("")
         self.verbose("Current time {}           : {}".format(args.utc and "UTC" or "   ", dt.strftime("%Y-%m-%d %H:%M:%S")))

zfs_autobackup/ZfsAutobackup.py

@@ -1,9 +1,7 @@
-import time
 
 import argparse
-from datetime import datetime
 from signal import signal, SIGPIPE
-from .util import output_redir, sigpipe_handler
+from .util import output_redir, sigpipe_handler, datetime_now
 
 from .ZfsAuto import ZfsAuto
 
@@ -203,7 +201,7 @@ class ZfsAutobackup(ZfsAuto):
                 else:
                     # past the deadline?
                     deadline_ttl = ThinnerRule("0s" + self.args.destroy_missing).ttl
-                    now = int(time.time())
+                    now = datetime_now(self.args.utc).timestamp()
                     if dataset.our_snapshots[-1].timestamp + deadline_ttl > now:
                         dataset.verbose("Destroy missing: Waiting for deadline.")
                     else:
@@ -461,8 +459,7 @@ class ZfsAutobackup(ZfsAuto):
             ################# snapshotting
             if not self.args.no_snapshot:
                 self.set_title("Snapshotting")
-                dt = datetime.utcnow() if self.args.utc else datetime.now()
-                snapshot_name = dt.strftime(self.snapshot_time_format)
+                snapshot_name = datetime_now(self.args.utc).strftime(self.snapshot_time_format)
                 source_node.consistent_snapshot(source_datasets, snapshot_name,
                                                 min_changed_bytes=self.args.min_change,
                                                 pre_snapshot_cmds=self.args.pre_snapshot_cmd,

zfs_autobackup/ZfsNode.py

@@ -12,6 +12,7 @@ from .CachedProperty import CachedProperty
 from .ZfsPool import ZfsPool
 from .ZfsDataset import ZfsDataset
 from .ExecuteNode import ExecuteError
+from .util import datetime_now
 
 
 class ZfsNode(ExecuteNode):
@@ -59,7 +60,8 @@ class ZfsNode(ExecuteNode):
     def thin(self, objects, keep_objects):
         # NOTE: if thinning is disabled with --no-thinning, self.__thinner will be none.
         if self.__thinner is not None:
-            return self.__thinner.thin(objects, keep_objects)
+
+            return self.__thinner.thin(objects, keep_objects, datetime_now(self.utc))
         else:
             return (keep_objects, [])
 

zfs_autobackup/test.py

@@ -1,129 +0,0 @@
-import os.path
-import os
-import subprocess
-import sys
-import time
-from signal import signal, SIGPIPE
-
-import util
-
-signal(SIGPIPE, util.sigpipe_handler)
-
-
-try:
-    print ("voor eerste")
-    raise Exception("eerstre")
-except Exception as e:
-    print ("voor tweede")
-    raise Exception("tweede")
-finally:
-    print ("JO")
-
-def generator():
-
-    try:
-        util.deb('in generator')
-        print ("TRIGGER SIGPIPE")
-        sys.stdout.flush()
-        util.deb('after trigger')
-
-        # if False:
-        yield ("bla")
-        # yield ("bla")
-
-    except GeneratorExit as e:
-        util.deb('GENEXIT '+str(e))
-        raise
-
-    except Exception as e:
-        util.deb('EXCEPT '+str(e))
-    finally:
-        util.deb('FINALLY')
-        print("nog iets")
-        sys.stdout.flush()
-        util.deb('after print in finally WOOP!')
-
-
-util.deb('START')
-g=generator()
-util.deb('after generator')
-for bla in g:
-    # print ("heb wat ontvangen")
-    util.deb('ontvangen van gen')
-    break
-    # raise Exception("moi")
-
-    pass
-raise Exception("moi")
-
-util.deb('after for')
-
-while True:
-    pass
-
-#
-# with open('test.py', 'rb') as fh:
-#
-#     # fsize = fh.seek(10000, os.SEEK_END)
-#     # print(fsize)
-#
-#     start=time.time()
-#     for i in range(0,1000000):
-#         # fh.seek(0, 0)
-#         fsize=fh.seek(0, os.SEEK_END)
-#         # fsize=fh.tell()
-#         # os.path.getsize('test.py')
-#     print(time.time()-start)
-#
-#
-#     print(fh.tell())
-#
-# sys.exit(0)
-#
-#
-#
-# checked=1
-# skipped=1
-# coverage=0.1
-#
-# max_skip=0
-#
-#
-# skipinarow=0
-# while True:
-#     total=checked+skipped
-#
-#     skip=coverage<random()
-#     if skip:
-#         skipped = skipped + 1
-#         print("S {:.2f}%".format(checked * 100 / total))
-#
-#         skipinarow = skipinarow+1
-#         if skipinarow>max_skip:
-#             max_skip=skipinarow
-#     else:
-#         skipinarow=0
-#         checked=checked+1
-#         print("C {:.2f}%".format(checked * 100 / total))
-#
-#     print(max_skip)
-#
-# skip=0
-# while True:
-#
-#     total=checked+skipped
-#     if skip>0:
-#         skip=skip-1
-#         skipped = skipped + 1
-#         print("S {:.2f}%".format(checked * 100 / total))
-#     else:
-#         checked=checked+1
-#         print("C {:.2f}%".format(checked * 100 / total))
-#
-#         #calc new skip
-#         skip=skip+((1/coverage)-1)*(random()*2)
-#         # print(skip)
-#         if skip> max_skip:
-#             max_skip=skip
-#
-#     print(max_skip)

zfs_autobackup/util.py

@@ -1,21 +1,9 @@
-# root@psyt14s:/home/psy/zfs_autobackup# ls -lh /home/psy/Downloads/carimage.zip
-# -rw-rw-r-- 1 psy psy 990M Nov 26  2020 /home/psy/Downloads/carimage.zip
-# root@psyt14s:/home/psy/zfs_autobackup# time sha1sum /home/psy/Downloads/carimage.zip
-# a682e1a36e16fe0d0c2f011104f4a99004f19105  /home/psy/Downloads/carimage.zip
-#
-# real	0m2.558s
-# user	0m2.105s
-# sys	0m0.448s
-# root@psyt14s:/home/psy/zfs_autobackup# time python3 -m zfs_autobackup.ZfsCheck
-#
-# real	0m1.459s
-# user	0m0.993s
-# sys	0m0.462s
 
 # NOTE: surprisingly sha1 in via python3 is faster than the native sha1sum utility, even in the way we use below!
 import os
 import platform
 import sys
+from datetime import datetime
 
 
 def tmp_name(suffix=""):
@@ -48,7 +36,7 @@ def output_redir():
 def sigpipe_handler(sig, stack):
     #redir output so we dont get more SIGPIPES during cleanup. (which my try to write to stdout)
     output_redir()
-    deb('redir')
+    #deb('redir')
 
 # def check_output():
 #     """make sure stdout still functions. if its broken, this will trigger a SIGPIPE which will be handled by the sigpipe_handler."""
@@ -63,3 +51,13 @@ def sigpipe_handler(sig, stack):
 #         fh.write("DEB: "+txt+"\n")
 
 
+# This should be the only source of trueth for the current datetime.
+# This function will be mocked during unit testing.
+
+
+datetime_now_mock=None
+def datetime_now(utc):
+    if datetime_now_mock is None:
+        return( datetime.utcnow() if utc else datetime.now())
+    else:
+        return datetime_now_mock