From 24dfb819c35269e1bf4cf85bf54e9c8340767ea0 Mon Sep 17 00:00:00 2001 From: Edwin Eefting Date: Wed, 28 Oct 2015 10:10:30 +0100 Subject: [PATCH] initial commit (from other repo) --- zfs_autobackup | 325 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100755 zfs_autobackup diff --git a/zfs_autobackup b/zfs_autobackup new file mode 100755 index 0000000..3853fbe --- /dev/null +++ b/zfs_autobackup @@ -0,0 +1,325 @@ +#!/usr/bin/env python +# -*- coding: utf8 -*- + +from __future__ import print_function +import os +import sys +import re +import traceback +import subprocess +import pprint +import cStringIO +import time + +###### parse arguments +import argparse +parser = argparse.ArgumentParser(description='ZFS autobackup v2.0') +parser.add_argument('--ssh-source', default="local", help='Source host to get backup from. (user@hostname) Default %(default)s.') +parser.add_argument('--ssh-target', default="local", help='Target host to push backup to. (user@hostname) Default %(default)s.') +parser.add_argument('--ssh-cipher', default="arcfour128", help='SSH cipher to use (default %(default)s)') +parser.add_argument('backup_name', help='Name of the backup (you should set the zfs property "autobackup:backup-name" to true on filesystems you want to backup') +parser.add_argument('target_fs', help='Target filesystem') + +parser.add_argument('--finish', action='store_true', help='dont create new snapshot, just finish sending current snapshots') +parser.add_argument('--compress', action='store_true', help='use compression during zfs send/recv') +parser.add_argument('--test', action='store_true', help='dont change anything, just show what would be done (still does all read-only operations)') +parser.add_argument('--verbose', action='store_true', help='verbose output') +parser.add_argument('--debug', action='store_true', help='debug output (shows user details, decicions that are made and commands that are executed)') +args = parser.parse_args() + + +def error(txt): + print(txt, file=sys.stderr) + + + +def verbose(txt): + if args.verbose: + print(txt) + + + +def debug(txt): + if args.debug: + print(txt) + + +"""run a command. specifiy ssh user@host to run remotely""" +def run(cmd, ssh_to=None, tab_split=False, valid_exitcodes=[ 0 ], test=False): + + encoded_cmd=[] + + #use ssh? + if ssh_to != "local": + encoded_cmd.extend(["ssh", "-c", args.ssh_cipher, ssh_to]) + if args.compress: + encoded_cmd.append("-C") + + #make sure the command gets all the data in utf8 format: + #(this is neccesary if LC_ALL=en_US.utf8 is not set in the environment) + for arg in cmd: + encoded_cmd.append(arg.encode('utf-8')) + + #the accurate way of displaying it whould be: print encoded_cmd + #However, we use the more human-readable way, but this is not always properly escaped! + #(most of the time it should be copypastable however.) + debug_txt="# "+encoded_cmd[0]+" '"+("' '".join(encoded_cmd[1:]))+"'" + + if test: + debug("[TEST] "+debug_txt) + return + else: + debug(debug_txt) + + p=subprocess.Popen(encoded_cmd, env=os.environ, stdout=subprocess.PIPE) + output=p.communicate()[0] + if p.returncode not in valid_exitcodes: + raise(subprocess.CalledProcessError(p.returncode, encoded_cmd)) + + lines=output.splitlines() + if not tab_split: + return(lines) + else: + ret=[] + for line in lines: + ret.append(line.split("\t")) + return(ret) + + + +"""determine filesystems that should be backupped by looking at the special autobackup-property""" +def zfs_get_selected_filesystems(ssh_to, backup_name): + #get all source filesystems that have the backup property + source_filesystems=run(ssh_to=ssh_to, tab_split=True, cmd=[ + "zfs", "get", "-t", "volume,filesystem", "-o", "name,value,source", "-s", "local,inherited", "-H", "autobackup:"+backup_name + ]) + + #determine filesystems that should be actually backupped + selected_filesystems=[] + direct_filesystems=[] + for source_filesystem in source_filesystems: + (name,value,source)=source_filesystem + if value=="false": + verbose("Ignoring: {0} (disabled)".format(name)) + + else: + if source=="local": + selected_filesystems.append(name) + direct_filesystems.append(name) + verbose("Selected: {0} (direct selection)".format(name)) + elif source.find("inherited from ")==0: + inherited_from=re.sub("^inherited from ", "", source) + if inherited_from in direct_filesystems: + selected_filesystems.append(name) + verbose("Selected: {0} (inherited selection)".format(name)) + else: + verbose("Ignored: {0} (already a backup)".format(name)) + else: + vebose("Ignored: {0} ({0})".format(source)) + + return(selected_filesystems) + + +#simulate snapshots for --test option +test_snapshots={} + + +"""create snapshot on multiple filesystems at once (atomicly)""" +def zfs_create_snapshot(ssh_to, filesystems, snapshot): + cmd=[ "zfs", "snapshot" ] + + for filesystem in filesystems: + cmd.append(filesystem+"@"+snapshot) + + #in testmode we dont actually make changes, so keep them in a list to simulate + if args.test: + if not ssh_to in test_snapshots: + test_snapshots[ssh_to]={} + if not filesystem in test_snapshots[ssh_to]: + test_snapshots[ssh_to][filesystem]=[] + test_snapshots[ssh_to][filesystem].append(snapshot) + + run(ssh_to=ssh_to, tab_split=False, cmd=cmd, test=args.test) + + + +"""get names of all snapshots for specified filesystems belonging to backup_name + +return[filesystem_name]=[ "snashot1", "snapshot2", ... ] +""" +def zfs_get_snapshots(ssh_to, filesystems, backup_name): + cmd=[ + "zfs", "list", "-d", "1", "-r", "-t" ,"snapshot", "-H", "-o", "name" + ] + cmd.extend(filesystems) + + snapshots=run(ssh_to=ssh_to, tab_split=False, cmd=cmd, valid_exitcodes=[ 0,1 ]) + + ret={} + for snapshot in snapshots: + (filesystem, snapshot_name)=snapshot.split("@") + if re.match("^"+backup_name+"-[0-9]*$", snapshot_name): + if not filesystem in ret: + ret[filesystem]=[] + ret[filesystem].append(snapshot_name) + + #also add any test-snapshots that where created with --test mode + if args.test: + if ssh_to in test_snapshots: + for filesystem in filesystems: + if filesystem in test_snapshots[ssh_to]: + ret[filesystem].extend(test_snapshots[ssh_to][filesystem]) + + return(ret) + + + +"""transfer a zfs snapshot from source to target. both can be either local or via ssh.""" +def zfs_transfer(ssh_source, source_filesystem, first_snapshot, second_snapshot, + ssh_target, target_filesystem): + + #### build source command + source_cmd=[] + + if ssh_source != "local": + source_cmd.extend([ "ssh", "-c", args.ssh_cipher, ssh_source ]) + if args.compress: + source_cmd.append("-C") + + source_cmd.extend(["zfs", "send", "-p" ]) + + if args.verbose: + source_cmd.append("-v") + + if not first_snapshot: + verbose("Tranferring "+source_filesystem+" initial backup snapshot "+second_snapshot) + else: + verbose("Tranferring "+source_filesystem+" incremental backup between snapshots "+first_snapshot+"..."+second_snapshot) + source_cmd.extend([ "-i", first_snapshot ]) + + source_cmd.append(source_filesystem + "@" + second_snapshot) + + + #### build target command + target_cmd=[] + + if ssh_target != "local": + target_cmd.extend([ "ssh", "-c", args.ssh_cipher, ssh_target ]) + if args.compress: + target_cmd.append("-C") + + target_cmd.extend(["zfs", "recv", "-u" ]) + + if args.verbose: + target_cmd.append("-v") + + target_cmd.append(target_filesystem) + + + #### make sure parent on target exists + parent_filesystem= "/".join(target_filesystem.split("/")[:-1]) + run(ssh_to=ssh_target, cmd=[ "zfs", "create" ,"-p", parent_filesystem ], test=args.test) + + + ### execute pipe + debug_txt="# "+source_cmd[0]+" '"+("' '".join(source_cmd[1:]))+"'" + " | " + target_cmd[0]+" '"+("' '".join(target_cmd[1:]))+"'" + + if args.test: + debug("[TEST] "+debug_txt) + return + else: + debug(debug_txt) + + source_proc=subprocess.Popen(source_cmd, env=os.environ, stdout=subprocess.PIPE) + target_proc=subprocess.Popen(target_cmd, env=os.environ, stdin=source_proc.stdout) + source_proc.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits. + target_proc.communicate() + + if source_proc.returncode: + raise(subprocess.CalledProcessError(source_proc.returncode, source_cmd)) + + #zfs recv sometimes gives an exitcode 1 while the transfer was succesfull, therefore we ignore exit 1's and do an extra check to see if the snapshot is there. + if target_proc.returncode and target_proc.returncode!=1: + raise(subprocess.CalledProcessError(target_proc.returncode, target_cmd)) + + debug("Verifying if snapshot exists on target") + run(ssh_to=ssh_target, cmd=["zfs", "list", target_filesystem+"@"+second_snapshot ]) + + + +################################################################## ENTRY POINT + +if args.test: + args.verbose=True + verbose("RUNNING IN TEST-MODE, NOT MAKING ACTUAL BACKUP!") + +#get selected filesystem on backup source +verbose("Getting selected source filesystems for backup {0} on {1}".format(args.backup_name,args.ssh_source)) +source_filesystems=zfs_get_selected_filesystems(args.ssh_source, args.backup_name) + +#nothing todo +if not source_filesystems: + error("No filesystems source selected, please set autobackup:{0} on {1}".format(args.backup_name,args.ssh_source)) + sys.exit(1) + + +#determine target filesystems +target_filesystems=[] +for source_filesystem in source_filesystems: + target_filesystems.append(args.target_fs+"/"+source_filesystem) + +#create new snapshot? +if not args.finish: + new_snapshot_name=args.backup_name+"-"+time.strftime("%Y%m%d%H%M%S") + verbose("Creating source snapshot {0} on {1} ".format(new_snapshot_name, args.ssh_source)) + zfs_create_snapshot(args.ssh_source, source_filesystems, new_snapshot_name) + +#determine all snapshots of all selected filesystems on both source and target +verbose("Getting source snapshot-list from {0}".format(args.ssh_source)) +source_snapshots=zfs_get_snapshots(args.ssh_source, source_filesystems, args.backup_name) +debug("Source snapshots: " + str(pprint.pformat(source_snapshots))) + +target_snapshots={} +try: + verbose("Getting target snapshot-list from {0}".format(args.ssh_target)) + target_snapshots=zfs_get_snapshots(args.ssh_target, target_filesystems, args.backup_name) +except subprocess.CalledProcessError: + verbose("(ignoring errors, probably initial backup for this filesystem)") + pass +debug("Target snapshots: " + str(pprint.pformat(target_snapshots))) + + +#determine which snapshots to send for each filesystem +for source_filesystem in source_filesystems: + target_filesystem=args.target_fs+"/"+source_filesystem + + if source_filesystem not in source_snapshots: + #this happens if you use --finish and there are new filesystems without snapshots + verbose("Skipping source filesystem {0}, no snapshots found".format(source_filesystem)) + else: + + send_snapshots=source_snapshots[source_filesystem][:] + + #remove snapshots that are already on target from send-list + if target_filesystem in target_snapshots and target_snapshots[target_filesystem]: + latest_target_snapshot=target_snapshots[target_filesystem][-1] + + if latest_target_snapshot not in send_snapshots: + raise(Exception("Cant find latest target snapshot on source, did you destroy it accidently? "+source_filesystem+"@"+latest_target_snapshot)) + + while latest_target_snapshot in send_snapshots: + send_snapshots.pop(0) + else: + latest_target_snapshot=None + + #now actually send the snapshots + for send_snapshot in send_snapshots: + zfs_transfer( + ssh_source=args.ssh_source, source_filesystem=source_filesystem, first_snapshot=latest_target_snapshot, second_snapshot=send_snapshot, + ssh_target=args.ssh_target, target_filesystem=target_filesystem) + + latest_target_snapshot=send_snapshot + + +verbose("All done") +