diff --git a/btv b/btv index abd69bc..206c75c 100755 --- a/btv +++ b/btv @@ -8,8 +8,11 @@ import datetime import json import os import shlex +import shutil import sys import time +import socket +import urllib.request CONFIG = "/etc/btv/config.ini" LOCKFILE = "/run/lock/btv/serialization.lock" @@ -196,9 +199,9 @@ def serialize(snap, outdir, key, snap_from=None): ## prepare directories ## if snap_from: - name = "%s diff from %s" %(snap.name, snap_from.name) + name = "%s to %s" %(snap_from.name, snap.name) else: - name = "%s full" %(snap.name) + name = snap.name directory = os.path.join(outdir, name) os.makedirs(directory) @@ -244,21 +247,37 @@ def serialize(snap, outdir, key, snap_from=None): ## final touches ## - ## add a self-check executable + ## add self-check and unpack executables with open(os.path.join(directory, "check-integrity.sh"), "w") as f: f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n") - os.chmod(f.name, 0o555) + os.chmod(f.name, 0o500) + + unpack_path = os.path.join(directory, "unpack.sh") + shutil.copy("/usr/share/btv/unpack.sh", unpack_path) + os.chmod(unpack_path, 0o500) ## fix permissions and ownership of created objects outdir_stat = os.stat(outdir) os.chown(directory, outdir_stat.st_uid, outdir_stat.st_gid) + os.chmod(directory, 0o700) for file in os.listdir(directory): path = os.path.join(directory, file) os.chown(path, outdir_stat.st_uid, outdir_stat.st_gid) + + if path.endswith(".aes") or path.endswith(".sha512"): + os.chmod(path, 0o400) + + return 0 +def ping(url): + try: + urllib.request.urlopen(url, timeout=10) + except socket.error as e: + print("Ping failed: %s" %(e)) + # ------------------------------------------------------------------------------ # Verbs @@ -278,30 +297,37 @@ def do_create(args): ## determine the rank of the new snapshot ## - snaps_since_rank_1 = 1 - snaps_since_rank_2 = 1 - - for snap in list_snapshots(): - if snap.rank == 1: - snaps_since_rank_1 = 1 - else: - snaps_since_rank_1 += 1 + if "--rank2" in args: + snapshot.rank = 2 + else: + snaps_since_rank_1 = 1 + snaps_since_rank_2 = 1 - if snap.rank == 2: - snaps_since_rank_1 = 1 - snaps_since_rank_2 = 1 - else: - snaps_since_rank_2 += 1 - - ## promote the snapshot - if snaps_since_rank_2 >= cfg.getint("snap", "rank_2_interval"): - if "--process" in args: - snapshot.rank = 2 - else: - print("!!! Rank 2 snapshot is due, please enable --process") + for snap in list_snapshots(): + if snap.rank == 1: + snaps_since_rank_1 = 1 + else: + snaps_since_rank_1 += 1 + + if snap.rank == 2: + snaps_since_rank_1 = 1 + snaps_since_rank_2 = 1 + else: + snaps_since_rank_2 += 1 + + ## promote the snapshot + if snaps_since_rank_2 >= cfg.getint("snap", "rank_2_interval"): + if "--process" in args: + snapshot.rank = 2 + else: + print("!!! Rank 2 snapshot is due, please enable --process") + snapshot.rank = 1 + elif snaps_since_rank_1 >= cfg.getint("snap", "rank_1_interval"): snapshot.rank = 1 - elif snaps_since_rank_1 >= cfg.getint("snap", "rank_1_interval"): - snapshot.rank = 1 + + ping_url = cfg.get("monitoring", "rank2_start_url") + if snapshot.rank == 2 and ping_url: + ping(ping_url) ## create the snapshot itself ## @@ -324,7 +350,6 @@ def do_create(args): ## do optional processing ## if snapshot.rank == 2: - ## snapshot serialization # if there's a previous Rank 2 snapshot, compute a diff against it # if not or if the process fails, demote this snap to Rank 1 @@ -356,6 +381,10 @@ def do_create(args): ## garbage collection do_gc() + + ping_url = cfg.get("monitoring", "rank2_end_url") + if snapshot.rank == 2 and ping_url: + ping(ping_url) def do_list(args): """ @@ -490,23 +519,42 @@ def do_gc(args=None): Drops old snapshots. If the only arg is "greedy", drops ALL snapshots except the youngest - Rank 2. + Rank 2. If it's a number, drops that many oldest snapshots. """ - if args and "greedy" in args: - newest = list_snapshots(2)[-1] - - if newest: - print(">>> Dropping all snapshots except the newest Rank 2 in 5 s...") - time.sleep(5) + if args: + if args[0] == "greedy": + newest = list_snapshots(2)[-1] - for snap in list_snapshots(): - if snap.name != newest.name: - snap.drop() + if newest: + print(">>> Dropping all snapshots except the newest Rank 2 in 5 s...") + time.sleep(5) + + for snap in list_snapshots(): + if snap.name != newest.name: + snap.drop() + + else: + print("!!! no Rank 2 snapshot exists") + sys.exit(1) else: - print("!!! no Rank 2 snapshot exists") - sys.exit(1) + try: + count = int(args[0]) + except ValueError: + print("!!! %s is not a count of snapshots to delete" %(args[0])) + sys.exit(1) + + snaps = list_snapshots()[:count] + + for snap in snaps: + print(" %d %s %s" %(snap.rank, snap.name, ", ".join(snap.notes))) + + print(">>> These snapshots will be dropped in 5 s...") + time.sleep(5) + + for snap in snaps: + snap.drop() else: counts = [0, 0, 0] # Rank 0, 1, 2 @@ -540,8 +588,10 @@ def print_help(): {cmd} VERB [args] Verbs: - snapshot [--process] - Create a snapshot. If --process is passed, do all optional work. + snapshot [--process] [--rank2] + Create a snapshot. If --process is passed, do all optional work. If + --rank2 is passed, the new snapshot is automatically promoted to Rank 2 + and --process is implied. stream full SNAPSHOT OUTPUT_DIR Streams the SNAPSHOT into OUTPUT_DIR as a full (milestone) bin. @@ -560,6 +610,9 @@ Verbs: gc greedy Drop ALL snapshots except the newest Rank 2. + + gc COUNT + Drop COUNT oldest snapshots. """.format(cmd=sys.argv[0])) if __name__ == "__main__": diff --git a/cfg/config.ini b/cfg/config.ini index cb00390..f3dc5a9 100644 --- a/cfg/config.ini +++ b/cfg/config.ini @@ -35,3 +35,10 @@ dir = /mnt/pool/subvol/_backup rank_0_count=25 rank_1_count=36 rank_2_count=42 + +[monitoring] +# GETs this URL before starting a rank2 snapdhot +rank2_start_url = + +# GETs this URL after successfully completing a rank2 snapshot +rank2_end_url = diff --git a/unpack.sh b/unpack.sh new file mode 100644 index 0000000..f78a8ba --- /dev/null +++ b/unpack.sh @@ -0,0 +1,24 @@ +#! /bin/zsh + +TIMESTAMP=($(basename "$(pwd)")) +OUTDIR="$1" +KEYFILE="$2" + +function die { + >&2 echo "$2" + exit $1 +} + +[[ "$0" != "./unpack.sh" ]] && die 1 "This can only be executed from the snapshot directory itself." +[[ ! -d "$OUTDIR" ]] && die 1 "The first argument must be a directory to unpack subvolumes into." +[[ ! -f "$KEYFILE" ]] && die 1 "The second argument must be a readable keyfile." +./check-integrity.sh || die 2 "This snapshot failed integrity checks." + +### end of checks + +for ARCHIVE in *btrfs.zst.aes +do + openssl enc -d -aes-256-cbc -pbkdf2 -salt -pass "file:$KEYFILE" < "$ARCHIVE" | zstd -d | btrfs receive "$OUTDIR" || die 3 "Failed to unpack subvolume." + SUBVOL_NAME=${ARCHIVE%%.btrfs.zst.aes} + mv "${OUTDIR}/${SUBVOL_NAME}" "${OUTDIR}/${SUBVOL_NAME}.${TIMESTAMP[1]}" || die 4 "Failed to rename subvolume." +done