forked from third-party-mirrors/zfs_autobackup
refactoring for oop and a better diff-engine
This commit is contained in:
Edwin Eefting
parent
8c121fab16
commit
441a323fb2
@ -23,6 +23,10 @@ It has the following features:
|
|||||||
* Easy installation:
|
* Easy installation:
|
||||||
* Only one host needs the zfs_autobackup script. The other host just needs ssh and the zfs command.
|
* Only one host needs the zfs_autobackup script. The other host just needs ssh and the zfs command.
|
||||||
* Written in python and uses zfs-commands, no 3rd party dependencys or libraries.
|
* Written in python and uses zfs-commands, no 3rd party dependencys or libraries.
|
||||||
|
* Tested on:
|
||||||
|
* SmartOS
|
||||||
|
* FreeNAS
|
||||||
|
* (others should work as well)
|
||||||
|
|
||||||
Usage
|
Usage
|
||||||
====
|
====
|
||||||
|
|||||||
265
zfs_autobackup
265
zfs_autobackup
@ -1,9 +1,5 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
# -*- coding: utf8 -*-
|
# -*- coding: utf8 -*-
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
from __future__ import print_function
|
from __future__ import print_function
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
@ -18,171 +14,170 @@ import time
|
|||||||
def error(txt):
|
def error(txt):
|
||||||
print(txt, file=sys.stderr)
|
print(txt, file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def verbose(txt):
|
def verbose(txt):
|
||||||
if args.verbose:
|
if args.verbose:
|
||||||
print(txt)
|
print(txt)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def debug(txt):
|
def debug(txt):
|
||||||
if args.debug:
|
if args.debug:
|
||||||
print(txt)
|
print(txt)
|
||||||
|
|
||||||
|
|
||||||
"""run a command. specifiy ssh user@host to run remotely"""
|
class Node(Object):
|
||||||
def run(cmd, input=None, ssh_to="local", tab_split=False, valid_exitcodes=[ 0 ], test=False):
|
"""an endpoint that contains zfs filesystems. can be local or remote"""
|
||||||
|
|
||||||
encoded_cmd=[]
|
def __init__(self, ssh_to='local'):
|
||||||
|
self.backup_name=backup_name
|
||||||
|
self.ssh_to=ssh_to
|
||||||
|
|
||||||
|
def run(cmd, input=None, tab_split=False, valid_exitcodes=[ 0 ], test=False):
|
||||||
|
"""run a command on the node"""
|
||||||
|
|
||||||
|
encoded_cmd=[]
|
||||||
|
|
||||||
|
#use ssh?
|
||||||
|
if self.ssh_to != "local":
|
||||||
|
encoded_cmd.extend(["ssh", self.ssh_to])
|
||||||
|
if args.ssh_cipher:
|
||||||
|
encoded_cmd.extend(["-c", args.ssh_cipher])
|
||||||
|
if args.compress:
|
||||||
|
encoded_cmd.append("-C")
|
||||||
|
|
||||||
|
#make sure the command gets all the data in utf8 format:
|
||||||
|
#(this is neccesary if LC_ALL=en_US.utf8 is not set in the environment)
|
||||||
|
for arg in cmd:
|
||||||
|
#add single quotes for remote commands to support spaces and other wierd stuff (remote commands are executed in a shell)
|
||||||
|
encoded_cmd.append( ("'"+arg+"'").encode('utf-8'))
|
||||||
|
|
||||||
|
else:
|
||||||
|
for arg in cmd:
|
||||||
|
encoded_cmd.append(arg.encode('utf-8'))
|
||||||
|
|
||||||
|
#debug and test stuff
|
||||||
|
debug_txt="# "+" ".join(encoded_cmd)
|
||||||
|
|
||||||
|
if test:
|
||||||
|
debug("[SKIPPING] "+debug_txt)
|
||||||
|
else:
|
||||||
|
debug(debug_txt)
|
||||||
|
|
||||||
|
if input:
|
||||||
|
debug("INPUT:\n"+input.rstrip())
|
||||||
|
stdin=subprocess.PIPE
|
||||||
|
else:
|
||||||
|
stdin=None
|
||||||
|
|
||||||
|
if test:
|
||||||
|
return
|
||||||
|
|
||||||
|
#execute and parse/return results
|
||||||
|
p=subprocess.Popen(encoded_cmd, env=os.environ, stdout=subprocess.PIPE, stdin=stdin)
|
||||||
|
output=p.communicate(input=input)[0]
|
||||||
|
if p.returncode not in valid_exitcodes:
|
||||||
|
raise(subprocess.CalledProcessError(p.returncode, encoded_cmd))
|
||||||
|
|
||||||
|
lines=output.splitlines()
|
||||||
|
if not tab_split:
|
||||||
|
return(lines)
|
||||||
|
else:
|
||||||
|
ret=[]
|
||||||
|
for line in lines:
|
||||||
|
ret.append(line.split("\t"))
|
||||||
|
return(ret)
|
||||||
|
|
||||||
|
|
||||||
#use ssh?
|
def zfs_get_selected_filesystems():
|
||||||
if ssh_to != "local":
|
"""determine filesystems that should be backupped by looking at the special autobackup-property"""
|
||||||
encoded_cmd.extend(["ssh", ssh_to])
|
|
||||||
if args.ssh_cipher:
|
#get all source filesystems that have the backup property
|
||||||
encoded_cmd.extend(["-c", args.ssh_cipher])
|
source_filesystems=self.run(tab_split=True, cmd=[
|
||||||
if args.compress:
|
"zfs", "get", "-t", "volume,filesystem", "-o", "name,value,source", "-s", "local,inherited", "-H", "autobackup:"+args.backup_name
|
||||||
encoded_cmd.append("-C")
|
])
|
||||||
|
|
||||||
|
#determine filesystems that should be actually backupped
|
||||||
|
selected_filesystems=[]
|
||||||
|
direct_filesystems=[]
|
||||||
|
for source_filesystem in source_filesystems:
|
||||||
|
(name,value,source)=source_filesystem
|
||||||
|
if value=="false":
|
||||||
|
verbose("Ignoring: {0} (disabled)".format(name))
|
||||||
|
|
||||||
|
else:
|
||||||
|
if source=="local":
|
||||||
|
selected_filesystems.append(name)
|
||||||
|
direct_filesystems.append(name)
|
||||||
|
verbose("Selected: {0} (direct selection)".format(name))
|
||||||
|
elif source.find("inherited from ")==0:
|
||||||
|
inherited_from=re.sub("^inherited from ", "", source)
|
||||||
|
if inherited_from in direct_filesystems:
|
||||||
|
selected_filesystems.append(name)
|
||||||
|
verbose("Selected: {0} (inherited selection)".format(name))
|
||||||
|
else:
|
||||||
|
verbose("Ignored: {0} (already a backup)".format(name))
|
||||||
|
else:
|
||||||
|
vebose("Ignored: {0} ({0})".format(source))
|
||||||
|
|
||||||
|
return(selected_filesystems)
|
||||||
|
|
||||||
|
|
||||||
#make sure the command gets all the data in utf8 format:
|
def zfs_get_resumable_filesystems(filesystems):
|
||||||
#(this is neccesary if LC_ALL=en_US.utf8 is not set in the environment)
|
"""determine filesystems that can be resumed via receive_resume_token (should be executed on target)"""
|
||||||
for arg in cmd:
|
|
||||||
#add single quotes for remote commands to support spaces and other wierd stuff (remote commands are executed in a shell)
|
|
||||||
encoded_cmd.append( ("'"+arg+"'").encode('utf-8'))
|
|
||||||
|
|
||||||
else:
|
cmd=[ "zfs", "get", "-t", "volume,filesystem", "-o", "name,value", "-H", "receive_resume_token" ]
|
||||||
for arg in cmd:
|
cmd.extend(filesystems)
|
||||||
encoded_cmd.append(arg.encode('utf-8'))
|
|
||||||
|
|
||||||
|
resumable_filesystems=self.run(tab_split=True, cmd=cmd)
|
||||||
|
|
||||||
#the accurate way of displaying it whould be: print encoded_cmd
|
ret={}
|
||||||
#However, we use the more human-readable way, but this is not always properly escaped!
|
|
||||||
#(most of the time it should be copypastable however.)
|
|
||||||
debug_txt="# "+" ".join(encoded_cmd)
|
|
||||||
|
|
||||||
if test:
|
for (resumable_filesystem,token) in resumable_filesystems:
|
||||||
debug("[TEST] "+debug_txt)
|
if token!='-':
|
||||||
else:
|
ret[resumable_filesystem]=token
|
||||||
debug(debug_txt)
|
|
||||||
|
|
||||||
if input:
|
|
||||||
debug("INPUT:\n"+input.rstrip())
|
|
||||||
stdin=subprocess.PIPE
|
|
||||||
else:
|
|
||||||
stdin=None
|
|
||||||
|
|
||||||
if test:
|
|
||||||
return
|
|
||||||
|
|
||||||
p=subprocess.Popen(encoded_cmd, env=os.environ, stdout=subprocess.PIPE, stdin=stdin)
|
|
||||||
output=p.communicate(input=input)[0]
|
|
||||||
if p.returncode not in valid_exitcodes:
|
|
||||||
raise(subprocess.CalledProcessError(p.returncode, encoded_cmd))
|
|
||||||
|
|
||||||
lines=output.splitlines()
|
|
||||||
if not tab_split:
|
|
||||||
return(lines)
|
|
||||||
else:
|
|
||||||
ret=[]
|
|
||||||
for line in lines:
|
|
||||||
ret.append(line.split("\t"))
|
|
||||||
return(ret)
|
return(ret)
|
||||||
|
|
||||||
|
|
||||||
"""determine filesystems that should be backupped by looking at the special autobackup-property"""
|
def zfs_destroy_snapshots(snapshots):
|
||||||
def zfs_get_selected_filesystems(ssh_to, backup_name):
|
"""deferred destroy list of snapshots (in @format). """
|
||||||
#get all source filesystems that have the backup property
|
|
||||||
source_filesystems=run(ssh_to=ssh_to, tab_split=True, cmd=[
|
|
||||||
"zfs", "get", "-t", "volume,filesystem", "-o", "name,value,source", "-s", "local,inherited", "-H", "autobackup:"+backup_name
|
|
||||||
])
|
|
||||||
|
|
||||||
#determine filesystems that should be actually backupped
|
#zfs can only destroy one filesystem at once so we use xargs and stdin
|
||||||
selected_filesystems=[]
|
self.run(test=args.test, input="\0".join(snapshots), cmd=
|
||||||
direct_filesystems=[]
|
[ "xargs", "-0", "-n", "1", "zfs", "destroy", "-d" ]
|
||||||
for source_filesystem in source_filesystems:
|
)
|
||||||
(name,value,source)=source_filesystem
|
|
||||||
if value=="false":
|
|
||||||
verbose("Ignoring: {0} (disabled)".format(name))
|
|
||||||
|
|
||||||
else:
|
|
||||||
if source=="local":
|
|
||||||
selected_filesystems.append(name)
|
|
||||||
direct_filesystems.append(name)
|
|
||||||
verbose("Selected: {0} (direct selection)".format(name))
|
|
||||||
elif source.find("inherited from ")==0:
|
|
||||||
inherited_from=re.sub("^inherited from ", "", source)
|
|
||||||
if inherited_from in direct_filesystems:
|
|
||||||
selected_filesystems.append(name)
|
|
||||||
verbose("Selected: {0} (inherited selection)".format(name))
|
|
||||||
else:
|
|
||||||
verbose("Ignored: {0} (already a backup)".format(name))
|
|
||||||
else:
|
|
||||||
vebose("Ignored: {0} ({0})".format(source))
|
|
||||||
|
|
||||||
return(selected_filesystems)
|
|
||||||
|
|
||||||
|
|
||||||
"""determine filesystems that can be resumed via receive_resume_token"""
|
def zfs_destroy(filesystems, recursive=False):
|
||||||
def zfs_get_resumable_filesystems(ssh_to, filesystems):
|
"""destroy list of filesystems """
|
||||||
|
|
||||||
cmd=[ "zfs", "get", "-t", "volume,filesystem", "-o", "name,value", "-H", "receive_resume_token" ]
|
cmd=[ "xargs", "-0", "-n", "1", "zfs", "destroy" ]
|
||||||
cmd.extend(filesystems)
|
|
||||||
|
|
||||||
#TODO: get rid of ugly errors for non-existing target filesystems
|
if recursive:
|
||||||
resumable_filesystems=run(ssh_to=ssh_to, tab_split=True, cmd=cmd, valid_exitcodes= [ 0,1 ] )
|
cmd.append("-r")
|
||||||
|
|
||||||
ret={}
|
#zfs can only destroy one filesystem at once so we use xargs and stdin
|
||||||
|
self.run(test=args.test, input="\0".join(filesystems), cmd=cmd)
|
||||||
for (resumable_filesystem,token) in resumable_filesystems:
|
|
||||||
if token!='-':
|
|
||||||
ret[resumable_filesystem]=token
|
|
||||||
|
|
||||||
return(ret)
|
|
||||||
|
|
||||||
|
|
||||||
"""deferred destroy list of snapshots (in @format). """
|
#simulate snapshots for --test option
|
||||||
def zfs_destroy_snapshots(ssh_to, snapshots):
|
#FIXME
|
||||||
|
test_snapshots={}
|
||||||
|
def zfs_create_snapshot(filesystems, snapshot):
|
||||||
|
"""create snapshot on multiple filesystems at once (atomicly)"""
|
||||||
|
|
||||||
#zfs can only destroy one filesystem at once so we use xargs and stdin
|
cmd=[ "zfs", "snapshot" ]
|
||||||
run(ssh_to=ssh_to, test=args.test, input="\0".join(snapshots), cmd=
|
|
||||||
[ "xargs", "-0", "-n", "1", "zfs", "destroy", "-d" ]
|
|
||||||
)
|
|
||||||
|
|
||||||
"""destroy list of filesystems """
|
for filesystem in filesystems:
|
||||||
def zfs_destroy(ssh_to, filesystems, recursive=False):
|
cmd.append(filesystem+"@"+snapshot)
|
||||||
|
|
||||||
cmd=[ "xargs", "-0", "-n", "1", "zfs", "destroy" ]
|
#in testmode we dont actually make changes, so keep them in a list to simulate
|
||||||
|
if args.test:
|
||||||
|
if not ssh_to in test_snapshots:
|
||||||
|
test_snapshots[ssh_to]={}
|
||||||
|
if not filesystem in test_snapshots[ssh_to]:
|
||||||
|
test_snapshots[ssh_to][filesystem]=[]
|
||||||
|
test_snapshots[ssh_to][filesystem].append(snapshot)
|
||||||
|
|
||||||
if recursive:
|
run(ssh_to=ssh_to, tab_split=False, cmd=cmd, test=args.test)
|
||||||
cmd.append("-r")
|
|
||||||
|
|
||||||
#zfs can only destroy one filesystem at once so we use xargs and stdin
|
|
||||||
run(ssh_to=ssh_to, test=args.test, input="\0".join(filesystems), cmd=cmd)
|
|
||||||
|
|
||||||
#simulate snapshots for --test option
|
|
||||||
test_snapshots={}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
"""create snapshot on multiple filesystems at once (atomicly)"""
|
|
||||||
def zfs_create_snapshot(ssh_to, filesystems, snapshot):
|
|
||||||
|
|
||||||
cmd=[ "zfs", "snapshot" ]
|
|
||||||
|
|
||||||
for filesystem in filesystems:
|
|
||||||
cmd.append(filesystem+"@"+snapshot)
|
|
||||||
|
|
||||||
#in testmode we dont actually make changes, so keep them in a list to simulate
|
|
||||||
if args.test:
|
|
||||||
if not ssh_to in test_snapshots:
|
|
||||||
test_snapshots[ssh_to]={}
|
|
||||||
if not filesystem in test_snapshots[ssh_to]:
|
|
||||||
test_snapshots[ssh_to][filesystem]=[]
|
|
||||||
test_snapshots[ssh_to][filesystem].append(snapshot)
|
|
||||||
|
|
||||||
run(ssh_to=ssh_to, tab_split=False, cmd=cmd, test=args.test)
|
|
||||||
|
|
||||||
|
|
||||||
"""get names of all snapshots for specified filesystems belonging to backup_name
|
"""get names of all snapshots for specified filesystems belonging to backup_name
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user