mirror of
https://github.com/psy0rz/zfs_autobackup.git
synced 2025-04-11 22:40:01 +03:00
407 lines
15 KiB
Python
Executable File
407 lines
15 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# -*- coding: utf8 -*-
|
|
|
|
from __future__ import print_function
|
|
import os
|
|
import sys
|
|
import re
|
|
import traceback
|
|
import subprocess
|
|
import pprint
|
|
import cStringIO
|
|
import time
|
|
|
|
###### parse arguments
|
|
import argparse
|
|
parser = argparse.ArgumentParser(description='ZFS autobackup v2.0')
|
|
parser.add_argument('--ssh-source', default="local", help='Source host to get backup from. (user@hostname) Default %(default)s.')
|
|
parser.add_argument('--ssh-target', default="local", help='Target host to push backup to. (user@hostname) Default %(default)s.')
|
|
parser.add_argument('--ssh-cipher', default="arcfour128", help='SSH cipher to use (default %(default)s)')
|
|
parser.add_argument('--keep-source', type=int, default=30, help='Number of old snapshots to keep on source. Default %(default)s.')
|
|
parser.add_argument('--keep-target', type=int, default=30, help='Number of old snapshots to keep on target. Default %(default)s.')
|
|
parser.add_argument('backup_name', help='Name of the backup (you should set the zfs property "autobackup:backup-name" to true on filesystems you want to backup')
|
|
parser.add_argument('target_fs', help='Target filesystem')
|
|
|
|
parser.add_argument('--finish', action='store_true', help='dont create new snapshot, just finish sending current snapshots')
|
|
parser.add_argument('--compress', action='store_true', help='use compression during zfs send/recv')
|
|
parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)')
|
|
parser.add_argument('--verbose', action='store_true', help='verbose output')
|
|
parser.add_argument('--debug', action='store_true', help='debug output (shows commands that are executed)')
|
|
args = parser.parse_args()
|
|
|
|
|
|
def error(txt):
|
|
print(txt, file=sys.stderr)
|
|
|
|
|
|
|
|
def verbose(txt):
|
|
if args.verbose:
|
|
print(txt)
|
|
|
|
|
|
|
|
def debug(txt):
|
|
if args.debug:
|
|
print(txt)
|
|
|
|
|
|
"""run a command. specifiy ssh user@host to run remotely"""
|
|
def run(cmd, input=None, ssh_to="local", tab_split=False, valid_exitcodes=[ 0 ], test=False):
|
|
|
|
encoded_cmd=[]
|
|
|
|
#use ssh?
|
|
if ssh_to != "local":
|
|
encoded_cmd.extend(["ssh", "-c", args.ssh_cipher, ssh_to])
|
|
if args.compress:
|
|
encoded_cmd.append("-C")
|
|
|
|
#make sure the command gets all the data in utf8 format:
|
|
#(this is neccesary if LC_ALL=en_US.utf8 is not set in the environment)
|
|
for arg in cmd:
|
|
encoded_cmd.append(arg.encode('utf-8'))
|
|
#the accurate way of displaying it whould be: print encoded_cmd
|
|
#However, we use the more human-readable way, but this is not always properly escaped!
|
|
#(most of the time it should be copypastable however.)
|
|
debug_txt="# "+encoded_cmd[0]+" '"+("' '".join(encoded_cmd[1:]))+"'"
|
|
|
|
if test:
|
|
debug("[TEST] "+debug_txt)
|
|
else:
|
|
debug(debug_txt)
|
|
|
|
if input:
|
|
debug("INPUT:\n"+input.rstrip())
|
|
stdin=subprocess.PIPE
|
|
else:
|
|
stdin=None
|
|
|
|
if test:
|
|
return
|
|
|
|
p=subprocess.Popen(encoded_cmd, env=os.environ, stdout=subprocess.PIPE, stdin=stdin)
|
|
output=p.communicate(input=input)[0]
|
|
if p.returncode not in valid_exitcodes:
|
|
raise(subprocess.CalledProcessError(p.returncode, encoded_cmd))
|
|
|
|
lines=output.splitlines()
|
|
if not tab_split:
|
|
return(lines)
|
|
else:
|
|
ret=[]
|
|
for line in lines:
|
|
ret.append(line.split("\t"))
|
|
return(ret)
|
|
|
|
|
|
|
|
"""determine filesystems that should be backupped by looking at the special autobackup-property"""
|
|
def zfs_get_selected_filesystems(ssh_to, backup_name):
|
|
#get all source filesystems that have the backup property
|
|
source_filesystems=run(ssh_to=ssh_to, tab_split=True, cmd=[
|
|
"zfs", "get", "-t", "volume,filesystem", "-o", "name,value,source", "-s", "local,inherited", "-H", "autobackup:"+backup_name
|
|
])
|
|
|
|
#determine filesystems that should be actually backupped
|
|
selected_filesystems=[]
|
|
direct_filesystems=[]
|
|
for source_filesystem in source_filesystems:
|
|
(name,value,source)=source_filesystem
|
|
if value=="false":
|
|
verbose("Ignoring: {0} (disabled)".format(name))
|
|
|
|
else:
|
|
if source=="local":
|
|
selected_filesystems.append(name)
|
|
direct_filesystems.append(name)
|
|
verbose("Selected: {0} (direct selection)".format(name))
|
|
elif source.find("inherited from ")==0:
|
|
inherited_from=re.sub("^inherited from ", "", source)
|
|
if inherited_from in direct_filesystems:
|
|
selected_filesystems.append(name)
|
|
verbose("Selected: {0} (inherited selection)".format(name))
|
|
else:
|
|
verbose("Ignored: {0} (already a backup)".format(name))
|
|
else:
|
|
vebose("Ignored: {0} ({0})".format(source))
|
|
|
|
return(selected_filesystems)
|
|
|
|
|
|
|
|
"""determine filesystems that where already backupped to the target. (used to determine cleanup list)"""
|
|
def zfs_get_backupped_filesystems(ssh_to, backup_name, target_fs):
|
|
#get all target filesystems that have the backup property and have a receive autobackup property
|
|
target_filesystems=run(ssh_to=ssh_to, tab_split=True, cmd=[
|
|
"zfs", "get", "-r", "-t", "volume,filesystem", "-o", "name,value,source", "-s", "received,inherited", "-H", "autobackup:"+backup_name, taget_fs
|
|
])
|
|
|
|
|
|
|
|
|
|
"""destroy list of filesystems or snapshots"""
|
|
def zfs_destroy(ssh_to, filesystems):
|
|
|
|
debug("Destroying on {0}:\n{1}".format(ssh_to, "\n".join(filesystems)))
|
|
#zfs can only destroy one filesystem at once so we use xargs and stdin
|
|
run(ssh_to=ssh_to, test=args.test, input="\0".join(filesystems), cmd=
|
|
[ "xargs", "-0", "-n", "1", "zfs", "destroy", "-d" ]
|
|
)
|
|
|
|
|
|
#simulate snapshots for --test option
|
|
test_snapshots={}
|
|
|
|
|
|
"""create snapshot on multiple filesystems at once (atomicly)"""
|
|
def zfs_create_snapshot(ssh_to, filesystems, snapshot):
|
|
cmd=[ "zfs", "snapshot" ]
|
|
|
|
for filesystem in filesystems:
|
|
cmd.append(filesystem+"@"+snapshot)
|
|
|
|
#in testmode we dont actually make changes, so keep them in a list to simulate
|
|
if args.test:
|
|
if not ssh_to in test_snapshots:
|
|
test_snapshots[ssh_to]={}
|
|
if not filesystem in test_snapshots[ssh_to]:
|
|
test_snapshots[ssh_to][filesystem]=[]
|
|
test_snapshots[ssh_to][filesystem].append(snapshot)
|
|
|
|
run(ssh_to=ssh_to, tab_split=False, cmd=cmd, test=args.test)
|
|
|
|
|
|
|
|
"""get names of all snapshots for specified filesystems belonging to backup_name
|
|
|
|
return[filesystem_name]=[ "snashot1", "snapshot2", ... ]
|
|
"""
|
|
def zfs_get_snapshots(ssh_to, filesystems, backup_name):
|
|
cmd=[
|
|
"zfs", "list", "-d", "1", "-r", "-t" ,"snapshot", "-H", "-o", "name"
|
|
]
|
|
cmd.extend(filesystems)
|
|
|
|
snapshots=run(ssh_to=ssh_to, tab_split=False, cmd=cmd, valid_exitcodes=[ 0,1 ])
|
|
|
|
ret={}
|
|
for snapshot in snapshots:
|
|
(filesystem, snapshot_name)=snapshot.split("@")
|
|
if re.match("^"+backup_name+"-[0-9]*$", snapshot_name):
|
|
if not filesystem in ret:
|
|
ret[filesystem]=[]
|
|
ret[filesystem].append(snapshot_name)
|
|
|
|
#also add any test-snapshots that where created with --test mode
|
|
if args.test:
|
|
if ssh_to in test_snapshots:
|
|
for filesystem in filesystems:
|
|
if filesystem in test_snapshots[ssh_to]:
|
|
ret[filesystem].extend(test_snapshots[ssh_to][filesystem])
|
|
|
|
return(ret)
|
|
|
|
|
|
|
|
"""transfer a zfs snapshot from source to target. both can be either local or via ssh.
|
|
|
|
specify buffer_size to use mbuffer (or alike) to apply buffering where neccesary
|
|
|
|
local to local:
|
|
local send -> local buffer -> local receive
|
|
|
|
local to remote and remote to local:
|
|
local send -> local buffer -> ssh -> remote buffer -> remote receive
|
|
remote send -> remote buffer -> ssh -> local buffer -> local receive
|
|
|
|
remote to remote:
|
|
remote send -> remote buffer -> ssh -> local buffer -> ssh -> remote buffer -> remote receive
|
|
"""
|
|
def zfs_transfer(ssh_source, source_filesystem, first_snapshot, second_snapshot,
|
|
ssh_target, target_filesystem, buffer_size=None):
|
|
|
|
#### build source command
|
|
source_cmd=[]
|
|
|
|
if ssh_source != "local":
|
|
source_cmd.extend([ "ssh", "-c", args.ssh_cipher, ssh_source ])
|
|
if args.compress:
|
|
source_cmd.append("-C")
|
|
|
|
source_cmd.extend(["zfs", "send", "-p" ])
|
|
|
|
if args.debug:
|
|
source_cmd.append("-v")
|
|
|
|
if not first_snapshot:
|
|
verbose("Tranferring "+source_filesystem+" initial backup snapshot "+second_snapshot)
|
|
else:
|
|
verbose("Tranferring "+source_filesystem+" incremental backup between snapshots "+first_snapshot+"..."+second_snapshot)
|
|
source_cmd.extend([ "-i", first_snapshot ])
|
|
|
|
source_cmd.append(source_filesystem + "@" + second_snapshot)
|
|
|
|
# if ssh_source != "local":
|
|
# #add buffer
|
|
# source_cmd.append("|dd")
|
|
|
|
|
|
#### build target command
|
|
target_cmd=[]
|
|
|
|
if ssh_target != "local":
|
|
target_cmd.extend([ "ssh", "-c", args.ssh_cipher, ssh_target ])
|
|
if args.compress:
|
|
target_cmd.append("-C")
|
|
|
|
target_cmd.extend(["zfs", "recv", "-u" ])
|
|
|
|
if args.verbose:
|
|
target_cmd.append("-v")
|
|
|
|
target_cmd.append(target_filesystem)
|
|
|
|
|
|
#### make sure parent on target exists
|
|
parent_filesystem= "/".join(target_filesystem.split("/")[:-1])
|
|
run(ssh_to=ssh_target, cmd=[ "zfs", "create" ,"-p", parent_filesystem ], test=args.test)
|
|
|
|
|
|
### execute pipe
|
|
debug_txt="# "+source_cmd[0]+" '"+("' '".join(source_cmd[1:]))+"'" + " | " + target_cmd[0]+" '"+("' '".join(target_cmd[1:]))+"'"
|
|
|
|
if args.test:
|
|
debug("[TEST] "+debug_txt)
|
|
return
|
|
else:
|
|
debug(debug_txt)
|
|
|
|
source_proc=subprocess.Popen(source_cmd, env=os.environ, stdout=subprocess.PIPE)
|
|
target_proc=subprocess.Popen(target_cmd, env=os.environ, stdin=source_proc.stdout)
|
|
source_proc.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits.
|
|
target_proc.communicate()
|
|
|
|
if source_proc.returncode:
|
|
raise(subprocess.CalledProcessError(source_proc.returncode, source_cmd))
|
|
|
|
#zfs recv sometimes gives an exitcode 1 while the transfer was succesfull, therefore we ignore exit 1's and do an extra check to see if the snapshot is there.
|
|
if target_proc.returncode and target_proc.returncode!=1:
|
|
raise(subprocess.CalledProcessError(target_proc.returncode, target_cmd))
|
|
|
|
debug("Verifying if snapshot exists on target")
|
|
run(ssh_to=ssh_target, cmd=["zfs", "list", target_filesystem+"@"+second_snapshot ])
|
|
|
|
|
|
|
|
################################################################## ENTRY POINT
|
|
|
|
if args.test:
|
|
args.verbose=True
|
|
verbose("RUNNING IN TEST-MODE, NOT MAKING ACTUAL BACKUP!")
|
|
|
|
if args.keep_source<1 or args.keep_target<1:
|
|
raise(Exception("Minimum number of snapshots to keep is 1"))
|
|
|
|
#get selected filesystem on backup source
|
|
verbose("Getting selected source filesystems for backup {0} on {1}".format(args.backup_name,args.ssh_source))
|
|
source_filesystems=zfs_get_selected_filesystems(args.ssh_source, args.backup_name)
|
|
|
|
#nothing todo
|
|
if not source_filesystems:
|
|
error("No filesystems source selected, please set autobackup:{0} on {1}".format(args.backup_name,args.ssh_source))
|
|
sys.exit(1)
|
|
|
|
|
|
#determine target filesystems
|
|
target_filesystems=[]
|
|
for source_filesystem in source_filesystems:
|
|
target_filesystems.append(args.target_fs+"/"+source_filesystem)
|
|
|
|
#create new snapshot?
|
|
if not args.finish:
|
|
new_snapshot_name=args.backup_name+"-"+time.strftime("%Y%m%d%H%M%S")
|
|
verbose("Creating source snapshot {0} on {1} ".format(new_snapshot_name, args.ssh_source))
|
|
zfs_create_snapshot(args.ssh_source, source_filesystems, new_snapshot_name)
|
|
|
|
#determine all snapshots of all selected filesystems on both source and target
|
|
verbose("Getting source snapshot-list from {0}".format(args.ssh_source))
|
|
source_snapshots=zfs_get_snapshots(args.ssh_source, source_filesystems, args.backup_name)
|
|
debug("Source snapshots: " + str(pprint.pformat(source_snapshots)))
|
|
|
|
target_snapshots={}
|
|
try:
|
|
verbose("Getting target snapshot-list from {0}".format(args.ssh_target))
|
|
target_snapshots=zfs_get_snapshots(args.ssh_target, target_filesystems, args.backup_name)
|
|
except subprocess.CalledProcessError:
|
|
verbose("(ignoring errors, probably initial backup for this filesystem)")
|
|
pass
|
|
debug("Target snapshots: " + str(pprint.pformat(target_snapshots)))
|
|
|
|
|
|
#determine which snapshots to send for each filesystem
|
|
for source_filesystem in source_filesystems:
|
|
target_filesystem=args.target_fs+"/"+source_filesystem
|
|
|
|
if source_filesystem not in source_snapshots:
|
|
#this happens if you use --finish and there are new filesystems without snapshots
|
|
verbose("Skipping source filesystem {0}, no snapshots found".format(source_filesystem))
|
|
else:
|
|
|
|
send_snapshots=source_snapshots[source_filesystem][:]
|
|
|
|
#remove snapshots that are already on target from send-list
|
|
if target_filesystem in target_snapshots and target_snapshots[target_filesystem]:
|
|
latest_target_snapshot=target_snapshots[target_filesystem][-1]
|
|
|
|
if latest_target_snapshot not in send_snapshots:
|
|
raise(Exception("Cant find latest target snapshot on source, did you destroy it accidently? "+source_filesystem+"@"+latest_target_snapshot))
|
|
|
|
while latest_target_snapshot in send_snapshots:
|
|
send_snapshots.pop(0)
|
|
else:
|
|
latest_target_snapshot=None
|
|
|
|
#now actually send the snapshots
|
|
for send_snapshot in send_snapshots:
|
|
zfs_transfer(
|
|
ssh_source=args.ssh_source, source_filesystem=source_filesystem, first_snapshot=latest_target_snapshot, second_snapshot=send_snapshot,
|
|
ssh_target=args.ssh_target, target_filesystem=target_filesystem)
|
|
|
|
#update target_snapshot list for later cleanup
|
|
if not target_filesystem in target_snapshots:
|
|
target_snapshots[target_filesystem]=[]
|
|
target_snapshots[target_filesystem].append(send_snapshot)
|
|
|
|
latest_target_snapshot=send_snapshot
|
|
|
|
|
|
#cleanup old source snapshots
|
|
source_destroys=[]
|
|
for source_filesystem in source_snapshots:
|
|
destroy_count=len(source_snapshots[source_filesystem])-args.keep_source
|
|
if destroy_count>0:
|
|
for snapshot in source_snapshots[source_filesystem][0:destroy_count-1]:
|
|
source_destroys.append(source_filesystem+"@"+snapshot)
|
|
|
|
if source_destroys:
|
|
verbose("Destroying old snapshots on source")
|
|
zfs_destroy(ssh_to=args.ssh_source, filesystems=source_destroys)
|
|
|
|
|
|
#cleanup old target snapshots
|
|
target_destroys=[]
|
|
for target_filesystem in target_snapshots:
|
|
destroy_count=len(target_snapshots[target_filesystem])-args.keep_target
|
|
if destroy_count>0:
|
|
for snapshot in target_snapshots[target_filesystem][0:destroy_count-1]:
|
|
target_destroys.append(target_filesystem+"@"+snapshot)
|
|
|
|
if target_destroys:
|
|
verbose("Destroying old snapshots on target")
|
|
zfs_destroy(ssh_to=args.ssh_target, filesystems=target_destroys)
|
|
|
|
|
|
verbose("All done")
|
|
|