forked from third-party-mirrors/zfs_autobackup
implemented --destroy-missing
This commit is contained in:
Edwin Eefting
parent
7b8b536d53
commit
9fe13a4207
@ -62,6 +62,8 @@ zfs-autobackup tries to be the easiest to use backup tool for zfs.
|
||||
* Uses **progressive thinning** for older snapshots.
|
||||
* Uses zfs-holds on important snapshots so they cant be accidentally destroyed.
|
||||
* Automatic resuming of failed transfers.
|
||||
* Can continue from existing common snapshots. (e.g. easy migration)
|
||||
* Gracefully handles destroyed datasets on source.
|
||||
* Easy installation:
|
||||
* Just install zfs-autobackup via pip, or download it manually.
|
||||
* Written in python and uses zfs-commands, no 3rd party dependency's or libraries.
|
||||
|
||||
@ -700,10 +700,11 @@ class ZfsDataset():
|
||||
|
||||
self.verbose("Destroying")
|
||||
|
||||
self.release()
|
||||
if self.is_snapshot:
|
||||
self.release()
|
||||
|
||||
try:
|
||||
self.zfs_node.run(["zfs", "destroy", "-d", self.name])
|
||||
self.zfs_node.run(["zfs", "destroy", self.name])
|
||||
self.invalidate()
|
||||
self.force_exists=False
|
||||
return(True)
|
||||
@ -898,12 +899,25 @@ class ZfsDataset():
|
||||
|
||||
@cached_property
|
||||
def recursive_datasets(self, types="filesystem,volume"):
|
||||
"""get all datasets recursively under us"""
|
||||
"""get all (non-snapshot) datasets recursively under us"""
|
||||
|
||||
self.debug("Getting all recursive datasets under us")
|
||||
|
||||
names=self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[ 0 ], cmd=[
|
||||
"zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name
|
||||
])
|
||||
|
||||
return(self.from_names(names[1:]))
|
||||
|
||||
|
||||
@cached_property
|
||||
def datasets(self, types="filesystem,volume"):
|
||||
"""get all (non-snapshot) datasets directly under us"""
|
||||
|
||||
self.debug("Getting all datasets under us")
|
||||
|
||||
names=self.zfs_node.run(tab_split=False, readonly=True, valid_exitcodes=[ 0 ], cmd=[
|
||||
"zfs", "list", "-r", "-t", types, "-o", "name", "-H", self.name
|
||||
"zfs", "list", "-r", "-t", types, "-o", "name", "-H", "-d", "1", self.name
|
||||
])
|
||||
|
||||
return(self.from_names(names[1:]))
|
||||
@ -1534,7 +1548,7 @@ class ZfsNode(ExecuteNode):
|
||||
selected_filesystems.append(dataset)
|
||||
dataset.verbose("Selected (inherited selection)")
|
||||
else:
|
||||
dataset.verbose("Ignored (already a backup)")
|
||||
dataset.debug("Ignored (already a backup)")
|
||||
else:
|
||||
dataset.verbose("Ignored (only childs)")
|
||||
|
||||
@ -1572,13 +1586,13 @@ class ZfsAutobackup:
|
||||
# parser.add_argument('--buffer', default="", help='Use mbuffer with specified size to speedup zfs transfer. (e.g. --buffer 1G) Will also show nice progress output.')
|
||||
|
||||
|
||||
# parser.add_argument('--destroy-stale', action='store_true', help='Destroy stale backups that have no more snapshots. Be sure to verify the output before using this! ')
|
||||
parser.add_argument('--clear-refreservation', action='store_true', help='Filter "refreservation" property. (recommended, safes space. same as --filter-properties refreservation)')
|
||||
parser.add_argument('--clear-mountpoint', action='store_true', help='Set property canmount=noauto for new datasets. (recommended, prevents mount conflicts. same as --set-properties canmount=noauto)')
|
||||
parser.add_argument('--filter-properties', type=str, help='List of properties to "filter" when receiving filesystems. (you can still restore them with zfs inherit -S)')
|
||||
parser.add_argument('--set-properties', type=str, help='List of propererties to override when receiving filesystems. (you can still restore them with zfs inherit -S)')
|
||||
parser.add_argument('--rollback', action='store_true', help='Rollback changes to the latest target snapshot before starting. (normally you can prevent changes by setting the readonly property on the target_path to on)')
|
||||
parser.add_argument('--destroy-incompatible', action='store_true', help='Destroy incompatible snapshots on target. Use with care! (implies --rollback)')
|
||||
parser.add_argument('--destroy-missing', type=str, default=None, help='Destroy datasets on target that are missing on the source. Specify the time since the last snapshot, e.g: --destroy-missing 30d')
|
||||
parser.add_argument('--ignore-transfer-errors', action='store_true', help='Ignore transfer errors (still checks if received filesystem exists. useful for acltype errors)')
|
||||
parser.add_argument('--raw', action='store_true', help='For encrypted datasets, send data exactly as it exists on disk.')
|
||||
|
||||
@ -1697,17 +1711,71 @@ class ZfsAutobackup:
|
||||
if self.args.debug:
|
||||
raise
|
||||
|
||||
#also thin target_datasets that are not on the source any more
|
||||
self.debug("Thinning obsolete datasets")
|
||||
for dataset in ZfsDataset(target_node, self.args.target_path).recursive_datasets:
|
||||
if dataset not in target_datasets:
|
||||
dataset.debug("Missing on source")
|
||||
dataset.thin()
|
||||
self.thin_missing_targets(ZfsDataset(target_node, self.args.target_path), target_datasets)
|
||||
|
||||
|
||||
return(fail_count)
|
||||
|
||||
|
||||
def thin_missing_targets(self, target_dataset, used_target_datasets):
|
||||
"""thin/destroy target datasets that are missing on the source."""
|
||||
|
||||
self.debug("Thinning obsolete datasets")
|
||||
|
||||
for dataset in target_dataset.recursive_datasets:
|
||||
try:
|
||||
if dataset not in used_target_datasets:
|
||||
dataset.debug("Missing on source, thinning")
|
||||
dataset.thin()
|
||||
|
||||
#destroy_missing enabled?
|
||||
if self.args.destroy_missing!=None:
|
||||
|
||||
#cant do anything without our own snapshots
|
||||
if not dataset.our_snapshots:
|
||||
if dataset.datasets:
|
||||
dataset.debug("Destroy missing: ignoring")
|
||||
else:
|
||||
dataset.verbose("Destroy missing: has no snapshots made by us. (please destroy manually)")
|
||||
else:
|
||||
#past the deadline?
|
||||
deadline_ttl=ThinnerRule("0s"+self.args.destroy_missing).ttl
|
||||
now=int(time.time())
|
||||
if dataset.our_snapshots[-1].timestamp + deadline_ttl > now:
|
||||
dataset.verbose("Destroy missing: Waiting for deadline.")
|
||||
else:
|
||||
|
||||
dataset.debug("Destroy missing: Removing our snapshots.")
|
||||
|
||||
#remove all our snaphots, except last, to safe space in case we fail later on
|
||||
for snapshot in dataset.our_snapshots[:-1]:
|
||||
snapshot.destroy(fail_exception=True)
|
||||
|
||||
#does it have other snapshots?
|
||||
has_others=False
|
||||
for snapshot in dataset.snapshots:
|
||||
if not snapshot.is_ours():
|
||||
has_others=True
|
||||
break
|
||||
|
||||
if has_others:
|
||||
dataset.verbose("Destroy missing: Still in use by other snapshots")
|
||||
else:
|
||||
if dataset.datasets:
|
||||
dataset.verbose("Destroy missing: Still has children here.")
|
||||
else:
|
||||
dataset.verbose("Destroy missing.")
|
||||
dataset.our_snapshots[-1].destroy(fail_exception=True)
|
||||
dataset.destroy(fail_exception=True)
|
||||
|
||||
except Exception as e:
|
||||
dataset.error("Error during destoy missing ({})".format(str(e)))
|
||||
|
||||
|
||||
return(fail_count)
|
||||
|
||||
|
||||
def thin_source(self, source_datasets):
|
||||
|
||||
self.set_title("Thinning source")
|
||||
|
||||
for source_dataset in source_datasets:
|
||||
@ -1752,7 +1820,7 @@ class ZfsAutobackup:
|
||||
self.set_title("Snapshotting")
|
||||
source_node.consistent_snapshot(source_datasets, source_node.new_snapshotname(), min_changed_bytes=self.args.min_change)
|
||||
|
||||
#if target is specified, we sync the datasets, otherwise we just thin the source.
|
||||
#if target is specified, we sync the datasets, otherwise we just thin the source. (e.g. snapshot mode)
|
||||
if self.args.target_path:
|
||||
fail_count=self.sync_datasets(source_node, source_datasets)
|
||||
else:
|
||||
|
||||
115
test_destroymissing.py
Normal file
115
test_destroymissing.py
Normal file
@ -0,0 +1,115 @@
|
||||
|
||||
from basetest import *
|
||||
|
||||
|
||||
class TestZfsNode(unittest2.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
prepare_zpools()
|
||||
self.longMessage=True
|
||||
|
||||
|
||||
|
||||
def test_destroymissing(self):
|
||||
|
||||
#initial backup
|
||||
with patch('time.strftime', return_value="10101111000000"): #1000 years in past
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-holds".split(" ")).run())
|
||||
|
||||
with patch('time.strftime', return_value="20101111000000"): #far in past
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-holds --allow-empty".split(" ")).run())
|
||||
|
||||
|
||||
with self.subTest("Should do nothing yet"):
|
||||
with OutputIO() as buf:
|
||||
with redirect_stdout(buf):
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 0s".split(" ")).run())
|
||||
|
||||
print(buf.getvalue())
|
||||
self.assertNotIn(": Destroy missing", buf.getvalue())
|
||||
|
||||
|
||||
with self.subTest("Normal destroyed leaf"):
|
||||
shelltest("zfs destroy -r test_source1/fs1/sub")
|
||||
|
||||
#wait for deadline of last snapshot
|
||||
with OutputIO() as buf:
|
||||
with redirect_stdout(buf):
|
||||
#100y: lastest should not be old enough, while second to latest snapshot IS old enough:
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 100y".split(" ")).run())
|
||||
|
||||
print(buf.getvalue())
|
||||
self.assertIn(": Waiting for deadline", buf.getvalue())
|
||||
|
||||
#past deadline, destroy
|
||||
with OutputIO() as buf:
|
||||
with redirect_stdout(buf):
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 1y".split(" ")).run())
|
||||
|
||||
print(buf.getvalue())
|
||||
self.assertIn("sub: Destroying", buf.getvalue())
|
||||
|
||||
|
||||
with self.subTest("Leaf with other snapshot still using it"):
|
||||
shelltest("zfs destroy -r test_source1/fs1")
|
||||
shelltest("zfs snapshot -r test_target1/test_source1/fs1@other1")
|
||||
|
||||
|
||||
with OutputIO() as buf:
|
||||
with redirect_stdout(buf):
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 0s".split(" ")).run())
|
||||
|
||||
print(buf.getvalue())
|
||||
#should have done the snapshot cleanup for destoy missing:
|
||||
self.assertIn("fs1@test-10101111000000: Destroying", buf.getvalue())
|
||||
#but cant finish because still in use:
|
||||
self.assertIn("fs1: Destroy missing: Still in use", buf.getvalue())
|
||||
|
||||
shelltest("zfs destroy test_target1/test_source1/fs1@other1")
|
||||
|
||||
|
||||
with self.subTest("In use by clone"):
|
||||
shelltest("zfs clone test_target1/test_source1/fs1@test-20101111000000 test_target1/clone1")
|
||||
|
||||
with OutputIO() as buf:
|
||||
with redirect_stdout(buf), redirect_stderr(buf):
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 0s".split(" ")).run())
|
||||
|
||||
print(buf.getvalue())
|
||||
#now tries to destroy our own last snapshot (before the final destroy of the dataset)
|
||||
self.assertIn("fs1@test-20101111000000: Destroying", buf.getvalue())
|
||||
#but cant finish because still in use:
|
||||
self.assertIn("fs1: Error during destoy missing", buf.getvalue())
|
||||
|
||||
shelltest("zfs destroy test_target1/clone1")
|
||||
|
||||
|
||||
with self.subTest("Should leave test_source1 parent"):
|
||||
|
||||
with OutputIO() as buf:
|
||||
with redirect_stdout(buf), redirect_stderr(buf):
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 0s".split(" ")).run())
|
||||
|
||||
print(buf.getvalue())
|
||||
#should have done the snapshot cleanup for destoy missing:
|
||||
self.assertIn("fs1: Destroying", buf.getvalue())
|
||||
|
||||
with OutputIO() as buf:
|
||||
with redirect_stdout(buf), redirect_stderr(buf):
|
||||
self.assertFalse(ZfsAutobackup("test test_target1 --verbose --no-snapshot --destroy-missing 0s".split(" ")).run())
|
||||
|
||||
print(buf.getvalue())
|
||||
#on second run it sees the dangling ex-parent but doesnt know what to do with it (since it has no own snapshot)
|
||||
self.assertIn("test_source1: Destroy missing: has no snapshots made by us.", buf.getvalue())
|
||||
|
||||
r=shelltest("zfs list -H -o name -r -t all test_target1")
|
||||
self.assertMultiLineEqual(r,"""
|
||||
test_target1
|
||||
test_target1/test_source1
|
||||
test_target1/test_source2
|
||||
test_target1/test_source2/fs2
|
||||
test_target1/test_source2/fs2/sub
|
||||
test_target1/test_source2/fs2/sub@test-10101111000000
|
||||
test_target1/test_source2/fs2/sub@test-20101111000000
|
||||
""")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user