diff --git a/btv b/btv index 206c75c..113b9b4 100755 --- a/btv +++ b/btv @@ -8,11 +8,7 @@ import datetime import json import os import shlex -import shutil import sys -import time -import socket -import urllib.request CONFIG = "/etc/btv/config.ini" LOCKFILE = "/run/lock/btv/serialization.lock" @@ -199,9 +195,9 @@ def serialize(snap, outdir, key, snap_from=None): ## prepare directories ## if snap_from: - name = "%s to %s" %(snap_from.name, snap.name) + name = "%s diff from %s" %(snap.name, snap_from.name) else: - name = snap.name + name = "%s full" %(snap.name) directory = os.path.join(outdir, name) os.makedirs(directory) @@ -217,10 +213,9 @@ def serialize(snap, outdir, key, snap_from=None): return 1 - ## serialization (most expensive) - ## with lockfile(LOCKFILE): - + ## serialize each subvolume or subvolume pair + ## for subvolume in snap.subvolumes: if snap_from: btrfs_send = 'btrfs send -p "%s" "%s"' %( @@ -232,52 +227,39 @@ def serialize(snap, outdir, key, snap_from=None): os.path.join(snap.path, subvolume) ) - error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" | hash-pipe sha512 "%s.btrfs.zst.aes" "%s/manifest.sha512" > "%s.btrfs.zst.aes"' %( + error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" > "%s.btrfs.zst.aes"' %( btrfs_send, cfg.get("crypto", "keyfile"), - subvolume, - directory, os.path.join(directory, subvolume) )) if error: print(" !! failed to serialize %s" %(subvolume)) return error - - ## final touches - ## - - ## add self-check and unpack executables - with open(os.path.join(directory, "check-integrity.sh"), "w") as f: - f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n") - os.chmod(f.name, 0o500) - - unpack_path = os.path.join(directory, "unpack.sh") - shutil.copy("/usr/share/btv/unpack.sh", unpack_path) - os.chmod(unpack_path, 0o500) + + ## calculate checksums and add a self-check executable + ## FIXME calculate this on the fly, re-reading is expensive + previous_wd = os.getcwd() + os.chdir(directory) + os.system('sha512sum "%s" > manifest.sha512' %( + '" "'.join("%s.btrfs.zst.aes" %(s) for s in snap.subvolumes) + )) + + with open("check-integrity.sh", "w") as f: + f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n") ## fix permissions and ownership of created objects + os.chmod("check-integrity.sh", 0o555) + outdir_stat = os.stat(outdir) os.chown(directory, outdir_stat.st_uid, outdir_stat.st_gid) - os.chmod(directory, 0o700) for file in os.listdir(directory): path = os.path.join(directory, file) os.chown(path, outdir_stat.st_uid, outdir_stat.st_gid) - - if path.endswith(".aes") or path.endswith(".sha512"): - os.chmod(path, 0o400) - - return 0 -def ping(url): - try: - urllib.request.urlopen(url, timeout=10) - except socket.error as e: - print("Ping failed: %s" %(e)) - # ------------------------------------------------------------------------------ # Verbs @@ -297,37 +279,30 @@ def do_create(args): ## determine the rank of the new snapshot ## - if "--rank2" in args: - snapshot.rank = 2 - else: - snaps_since_rank_1 = 1 - snaps_since_rank_2 = 1 - - for snap in list_snapshots(): - if snap.rank == 1: - snaps_since_rank_1 = 1 - else: - snaps_since_rank_1 += 1 - - if snap.rank == 2: - snaps_since_rank_1 = 1 - snaps_since_rank_2 = 1 - else: - snaps_since_rank_2 += 1 - - ## promote the snapshot - if snaps_since_rank_2 >= cfg.getint("snap", "rank_2_interval"): - if "--process" in args: - snapshot.rank = 2 - else: - print("!!! Rank 2 snapshot is due, please enable --process") - snapshot.rank = 1 - elif snaps_since_rank_1 >= cfg.getint("snap", "rank_1_interval"): - snapshot.rank = 1 + snaps_since_rank_1 = 1 + snaps_since_rank_2 = 1 - ping_url = cfg.get("monitoring", "rank2_start_url") - if snapshot.rank == 2 and ping_url: - ping(ping_url) + for snap in list_snapshots(): + if snap.rank == 1: + snaps_since_rank_1 = 1 + else: + snaps_since_rank_1 += 1 + + if snap.rank == 2: + snaps_since_rank_1 = 1 + snaps_since_rank_2 = 1 + else: + snaps_since_rank_2 += 1 + + ## promote the snapshot + if snaps_since_rank_2 >= cfg.getint("snap", "rank_2_interval"): + if "--process" in args: + snapshot.rank = 2 + else: + print("!!! Rank 2 snapshot is due, please enable --process") + snapshot.rank = 1 + elif snaps_since_rank_1 >= cfg.getint("snap", "rank_1_interval"): + snapshot.rank = 1 ## create the snapshot itself ## @@ -350,6 +325,7 @@ def do_create(args): ## do optional processing ## if snapshot.rank == 2: + ## snapshot serialization # if there's a previous Rank 2 snapshot, compute a diff against it # if not or if the process fails, demote this snap to Rank 1 @@ -373,18 +349,14 @@ def do_create(args): else: print("!!! no previous Rank 2 snapshot, please create one using btv stream") snapshot.rank = 1 + + ## garbage collection + do_gc() ## save all snapshot metadata snapshot.dump() print(">>> Snapshot created: rank %d %s" %(snapshot.rank, snapshot.name)) - - ## garbage collection - do_gc() - - ping_url = cfg.get("monitoring", "rank2_end_url") - if snapshot.rank == 2 and ping_url: - ping(ping_url) def do_list(args): """ @@ -517,60 +489,22 @@ def do_stream(args): def do_gc(args=None): """ Drops old snapshots. - - If the only arg is "greedy", drops ALL snapshots except the youngest - Rank 2. If it's a number, drops that many oldest snapshots. """ - if args: - if args[0] == "greedy": - newest = list_snapshots(2)[-1] - - if newest: - print(">>> Dropping all snapshots except the newest Rank 2 in 5 s...") - time.sleep(5) - - for snap in list_snapshots(): - if snap.name != newest.name: - snap.drop() - - else: - print("!!! no Rank 2 snapshot exists") - sys.exit(1) - - else: - try: - count = int(args[0]) - except ValueError: - print("!!! %s is not a count of snapshots to delete" %(args[0])) - sys.exit(1) - - snaps = list_snapshots()[:count] - - for snap in snaps: - print(" %d %s %s" %(snap.rank, snap.name, ", ".join(snap.notes))) - - print(">>> These snapshots will be dropped in 5 s...") - time.sleep(5) - - for snap in snaps: - snap.drop() + counts = [0, 0, 0] # Rank 0, 1, 2 + limits = [ + cfg.getint("gc", "rank_0_count"), + cfg.getint("gc", "rank_1_count"), + cfg.getint("gc", "rank_2_count") + ] - else: - counts = [0, 0, 0] # Rank 0, 1, 2 - limits = [ - cfg.getint("gc", "rank_0_count"), - cfg.getint("gc", "rank_1_count"), - cfg.getint("gc", "rank_2_count") - ] + for snap in reversed(list_snapshots()): + counts[snap.rank] += 1 - for snap in reversed(list_snapshots()): - counts[snap.rank] += 1 + if counts[snap.rank] > limits[snap.rank]: + print(" >> delete Rank %d %s" %(snap.rank, snap.name)) - if counts[snap.rank] > limits[snap.rank]: - print(" >> delete Rank %d %s" %(snap.rank, snap.name)) - - snap.drop() + snap.drop() verb_router = { "snapshot": do_create, @@ -588,10 +522,8 @@ def print_help(): {cmd} VERB [args] Verbs: - snapshot [--process] [--rank2] - Create a snapshot. If --process is passed, do all optional work. If - --rank2 is passed, the new snapshot is automatically promoted to Rank 2 - and --process is implied. + snapshot [--process] + Create a snapshot. If --process is passed, do all optional work. stream full SNAPSHOT OUTPUT_DIR Streams the SNAPSHOT into OUTPUT_DIR as a full (milestone) bin. @@ -607,12 +539,6 @@ Verbs: gc Drops old local snapshots based on garbage collector settings. - - gc greedy - Drop ALL snapshots except the newest Rank 2. - - gc COUNT - Drop COUNT oldest snapshots. """.format(cmd=sys.argv[0])) if __name__ == "__main__": diff --git a/cfg/config.ini b/cfg/config.ini index f3dc5a9..f5b72b0 100644 --- a/cfg/config.ini +++ b/cfg/config.ini @@ -30,15 +30,9 @@ dir = /mnt/pool/subvol/_backup [gc] # How many snapshots of each Rank to keep around. +# Note: pruning is only performed when a new Rank 2 snapshot is attempted. # Default values are 25, 36, and 42, i.e. 5 hours for Rank 0, 2 days for Rank 1, # and a week for Rank 2. rank_0_count=25 rank_1_count=36 rank_2_count=42 - -[monitoring] -# GETs this URL before starting a rank2 snapdhot -rank2_start_url = - -# GETs this URL after successfully completing a rank2 snapshot -rank2_end_url = diff --git a/unpack.sh b/unpack.sh deleted file mode 100644 index f78a8ba..0000000 --- a/unpack.sh +++ /dev/null @@ -1,24 +0,0 @@ -#! /bin/zsh - -TIMESTAMP=($(basename "$(pwd)")) -OUTDIR="$1" -KEYFILE="$2" - -function die { - >&2 echo "$2" - exit $1 -} - -[[ "$0" != "./unpack.sh" ]] && die 1 "This can only be executed from the snapshot directory itself." -[[ ! -d "$OUTDIR" ]] && die 1 "The first argument must be a directory to unpack subvolumes into." -[[ ! -f "$KEYFILE" ]] && die 1 "The second argument must be a readable keyfile." -./check-integrity.sh || die 2 "This snapshot failed integrity checks." - -### end of checks - -for ARCHIVE in *btrfs.zst.aes -do - openssl enc -d -aes-256-cbc -pbkdf2 -salt -pass "file:$KEYFILE" < "$ARCHIVE" | zstd -d | btrfs receive "$OUTDIR" || die 3 "Failed to unpack subvolume." - SUBVOL_NAME=${ARCHIVE%%.btrfs.zst.aes} - mv "${OUTDIR}/${SUBVOL_NAME}" "${OUTDIR}/${SUBVOL_NAME}.${TIMESTAMP[1]}" || die 4 "Failed to rename subvolume." -done