From cdac2e8ab41d637f2848f750845a8c3cd825bb34 Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Fri, 28 Aug 2020 23:54:51 +0200 Subject: [PATCH 01/16] btv 0.3.0 When serializing, create a lockfile which prevents the automatic snapshot service from starting. Add diff mode to the stream command. --- btv | 174 ++++++++++++++++++++++++++----------- systemd/btv-backup.service | 1 + 2 files changed, 126 insertions(+), 49 deletions(-) diff --git a/btv b/btv index d75727f..113b9b4 100755 --- a/btv +++ b/btv @@ -11,6 +11,7 @@ import shlex import sys CONFIG = "/etc/btv/config.ini" +LOCKFILE = "/run/lock/btv/serialization.lock" # ------------------------------------------------------------------------------ # Global @@ -33,6 +34,20 @@ def chdir(new_dir): finally: os.chdir(previous_dir) +@contextmanager +def lockfile(path): + d = os.path.dirname(path) + + if not os.path.exists(d): + os.makedirs(d) + + f = open(path, "w") + + try: + yield + finally: + os.remove(path) + # ------------------------------------------------------------------------------ # Functions @@ -172,6 +187,8 @@ def serialize(snap, outdir, key, snap_from=None): Snap and snap_from are Snapshot objects. + A lockfile is kept for the duration of the process. + Returns 0 on success. """ @@ -196,39 +213,40 @@ def serialize(snap, outdir, key, snap_from=None): return 1 - ## serialize each subvolume or subvolume pair - ## - for subvolume in snap.subvolumes: - if snap_from: - btrfs_send = 'btrfs send -p "%s" "%s"' %( - os.path.join(snap_from.path, subvolume), - os.path.join(snap.path, subvolume) - ) - else: - btrfs_send = 'btrfs send "%s"' %( - os.path.join(snap.path, subvolume) - ) + with lockfile(LOCKFILE): + ## serialize each subvolume or subvolume pair + ## + for subvolume in snap.subvolumes: + if snap_from: + btrfs_send = 'btrfs send -p "%s" "%s"' %( + os.path.join(snap_from.path, subvolume), + os.path.join(snap.path, subvolume) + ) + else: + btrfs_send = 'btrfs send "%s"' %( + os.path.join(snap.path, subvolume) + ) + + error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" > "%s.btrfs.zst.aes"' %( + btrfs_send, + cfg.get("crypto", "keyfile"), + os.path.join(directory, subvolume) + )) + + if error: + print(" !! failed to serialize %s" %(subvolume)) + return error - error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" > "%s.btrfs.zst.aes"' %( - btrfs_send, - cfg.get("crypto", "keyfile"), - os.path.join(directory, subvolume) + ## calculate checksums and add a self-check executable + ## FIXME calculate this on the fly, re-reading is expensive + previous_wd = os.getcwd() + os.chdir(directory) + os.system('sha512sum "%s" > manifest.sha512' %( + '" "'.join("%s.btrfs.zst.aes" %(s) for s in snap.subvolumes) )) - if error: - print(" !! failed to serialize %s" %(subvolume)) - return error - - ## calculate checksums and add a self-check executable - ## FIXME calculate this on the fly, re-reading is expensive - previous_wd = os.getcwd() - os.chdir(directory) - os.system('sha512sum "%s" > manifest.sha512' %( - '" "'.join("%s.btrfs.zst.aes" %(s) for s in snap.subvolumes) - )) - - with open("check-integrity.sh", "w") as f: - f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n") + with open("check-integrity.sh", "w") as f: + f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n") ## fix permissions and ownership of created objects os.chmod("check-integrity.sh", 0o555) @@ -380,39 +398,93 @@ def do_drop(args): def do_stream(args): """ - Stream the snapshot args[0] into dir args[1]. + args are either - The affected snapshot is then promoted to Rank 2. + "diff" SNAPSHOT_FROM SNAPSHOT_TO OUTPUT_DIR + + or + + "full" SNAPSHOT OUTPUT_DIR + + SNAPSHOT_FROM must be Rank 2. + + Streams the full or diff snapshot into OUTPUT_DIR. + + The SNAPSHOT or SNAPSHOT_TO is then promoted to Rank 2. """ - snapshot = get_snapshot_by_name(args[0]) + ## args interpretation and validation + ## + if not args: + raise UsageError("no args") - if not snapshot: - print("!!! %s is not a snapshot" %(args[0])) + if args[0] == "diff": + if len(args) != 4: + raise UsageError("'stream diff' requires exactly 3 arguments") + + snap_from_name, snap_name, output_dir = args[1:] + snap_from = get_snapshot_by_name(snap_from_name) + + if not snap_from: + print("!!! %s is not a snapshot" %(snap_from_name)) + sys.exit(2) + + if snap_from.rank != 2: + print("!!! source snapshot must be Rank 2, %s is %d" %(snap_from.name, snap_from.rank)) + print(' > Hint: btv stream full %s "%s"' %(snap_from.name, output_dir)) + sys.exit(3) + + elif args[0] == "full": + if len(args) != 3: + raise UsageError("'stream full' requires exactly 2 arguments") + + snap_name, output_dir = args[1:] + snap_from = None + + else: + raise UsageError("'stream' type is either 'full' or 'diff'") + + snap = get_snapshot_by_name(snap_name) + + if not snap: + print("!!! %s is not a snapshot" %(snap_name)) sys.exit(2) - directory = args[1] - - if not os.path.isdir(directory): - print("!!! %s is not a directory" %(directory)) + if not os.path.isdir(output_dir): + print("!!! %s is not a directory" %(output_dir)) sys.exit(2) - print(">>> Serializing %s into %s" %(snapshot.name, directory)) + if snap_from and snap_from.name >= snap.name: + print("!!! source snapshot is younger than target snapshot") + sys.exit(3) + + ## serialization work + ## + if snap_from: + comment = "diff %s -> %s" %(snap_from.name, snap.name) + else: + comment = "full %s" %(snap.name) + + print(">>> Serializing %s into %s" %(comment, output_dir)) error = serialize( - snapshot, - directory, - cfg.get("crypto", "keyfile") + snap, + output_dir, + cfg.get("crypto", "keyfile"), + snap_from ) if error: print("!!! serialization failed") else: - snapshot.rank = 2 - snapshot.notes.add("fully streamed") - snapshot.dump() + snap.rank = 2 + snap.notes.add("manually streamed") - print("<<< %s serialized and promoted to Rank %d" %(snapshot.name, snapshot.rank)) + if not snap_from: + snap.notes.add("fully streamed") + snap.dump() + + print("<<< %s serialized and promoted to Rank 2" %(snap.name)) def do_gc(args=None): """ @@ -453,9 +525,12 @@ Verbs: snapshot [--process] Create a snapshot. If --process is passed, do all optional work. - stream SNAPSHOT OUTPUT_DIR + stream full SNAPSHOT OUTPUT_DIR Streams the SNAPSHOT into OUTPUT_DIR as a full (milestone) bin. + stream diff SNAPSHOT_FROM SNAPSHOT_TO OUTPUT_DIR + Streams the difference between SNAPSHOTs into OUTPUT_DIR as a diff bin. + list List snapshots and their rank. @@ -480,7 +555,8 @@ if __name__ == "__main__": try: verb_router[verb](sys.argv[2:]) - except UsageError: + except UsageError as e: + print("!!! %s" %(e)) print() print_help() sys.exit(1) diff --git a/systemd/btv-backup.service b/systemd/btv-backup.service index 19eb0cb..d63bec8 100644 --- a/systemd/btv-backup.service +++ b/systemd/btv-backup.service @@ -1,5 +1,6 @@ [Unit] Description=Filesystem snapshot (with offsite backup) +ConditionPathExists=!/run/lock/btv/serialization.lock [Service] Type=oneshot From 5dda6562a6921226609ae035c73788b6f8cb1252 Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Sat, 29 Aug 2020 02:16:50 +0200 Subject: [PATCH 02/16] calculate hashes on-the-fly (major speedup) --- btv | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/btv b/btv index 113b9b4..43570ad 100755 --- a/btv +++ b/btv @@ -213,9 +213,10 @@ def serialize(snap, outdir, key, snap_from=None): return 1 + ## serialization (most expensive) + ## with lockfile(LOCKFILE): - ## serialize each subvolume or subvolume pair - ## + for subvolume in snap.subvolumes: if snap_from: btrfs_send = 'btrfs send -p "%s" "%s"' %( @@ -227,30 +228,27 @@ def serialize(snap, outdir, key, snap_from=None): os.path.join(snap.path, subvolume) ) - error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" > "%s.btrfs.zst.aes"' %( + error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" | hash-pipe sha512 "%s.btrfs.zst.aes" "%s/manifest.sha512" > "%s.btrfs.zst.aes"' %( btrfs_send, cfg.get("crypto", "keyfile"), + subvolume, + directory, os.path.join(directory, subvolume) )) if error: print(" !! failed to serialize %s" %(subvolume)) return error - - ## calculate checksums and add a self-check executable - ## FIXME calculate this on the fly, re-reading is expensive - previous_wd = os.getcwd() - os.chdir(directory) - os.system('sha512sum "%s" > manifest.sha512' %( - '" "'.join("%s.btrfs.zst.aes" %(s) for s in snap.subvolumes) - )) - - with open("check-integrity.sh", "w") as f: - f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n") + + ## final touches + ## + + ## add a self-check executable + with open(os.path.join(directory, "check-integrity.sh"), "w") as f: + f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n") + os.chmod(f.name, 0o555) ## fix permissions and ownership of created objects - os.chmod("check-integrity.sh", 0o555) - outdir_stat = os.stat(outdir) os.chown(directory, outdir_stat.st_uid, outdir_stat.st_gid) From ad19226d023a329fa3a284971983e6ff011bd236 Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Sat, 29 Aug 2020 02:42:54 +0200 Subject: [PATCH 03/16] add gc greedy --- btv | 47 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/btv b/btv index 43570ad..99549ed 100755 --- a/btv +++ b/btv @@ -9,6 +9,7 @@ import json import os import shlex import sys +import time CONFIG = "/etc/btv/config.ini" LOCKFILE = "/run/lock/btv/serialization.lock" @@ -487,22 +488,41 @@ def do_stream(args): def do_gc(args=None): """ Drops old snapshots. + + If the only arg is "greedy", drops ALL snapshots except the youngest + Rank 2. """ - counts = [0, 0, 0] # Rank 0, 1, 2 - limits = [ - cfg.getint("gc", "rank_0_count"), - cfg.getint("gc", "rank_1_count"), - cfg.getint("gc", "rank_2_count") - ] - - for snap in reversed(list_snapshots()): - counts[snap.rank] += 1 + if args and "greedy" in args: + newest = list_snapshots(2)[-1] - if counts[snap.rank] > limits[snap.rank]: - print(" >> delete Rank %d %s" %(snap.rank, snap.name)) + if newest: + print(">>> Dropping all snapshots except the newest Rank 2 in 5 s...") + time.sleep(5) - snap.drop() + for snap in list_snapshots(): + if snap.name != newest.name: + snap.drop() + + else: + print("!!! no Rank 2 snapshot exists") + sys.exit(1) + + else: + counts = [0, 0, 0] # Rank 0, 1, 2 + limits = [ + cfg.getint("gc", "rank_0_count"), + cfg.getint("gc", "rank_1_count"), + cfg.getint("gc", "rank_2_count") + ] + + for snap in reversed(list_snapshots()): + counts[snap.rank] += 1 + + if counts[snap.rank] > limits[snap.rank]: + print(" >> delete Rank %d %s" %(snap.rank, snap.name)) + + snap.drop() verb_router = { "snapshot": do_create, @@ -537,6 +557,9 @@ Verbs: gc Drops old local snapshots based on garbage collector settings. + + gc greedy + Drop ALL snapshots except the newest Rank 2. """.format(cmd=sys.argv[0])) if __name__ == "__main__": From cd2458b8303d37c66347e3e9f3cf3a592f03483c Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Tue, 1 Sep 2020 12:22:57 +0200 Subject: [PATCH 04/16] fix gc exceeding limits --- btv | 6 +++--- cfg/config.ini | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/btv b/btv index 99549ed..da0ad2f 100755 --- a/btv +++ b/btv @@ -348,9 +348,9 @@ def do_create(args): else: print("!!! no previous Rank 2 snapshot, please create one using btv stream") snapshot.rank = 1 - - ## garbage collection - do_gc() + + ## garbage collection + do_gc() ## save all snapshot metadata snapshot.dump() diff --git a/cfg/config.ini b/cfg/config.ini index f5b72b0..cb00390 100644 --- a/cfg/config.ini +++ b/cfg/config.ini @@ -30,7 +30,6 @@ dir = /mnt/pool/subvol/_backup [gc] # How many snapshots of each Rank to keep around. -# Note: pruning is only performed when a new Rank 2 snapshot is attempted. # Default values are 25, 36, and 42, i.e. 5 hours for Rank 0, 2 days for Rank 1, # and a week for Rank 2. rank_0_count=25 From aa96fa3cf53acbc4c5a3cca96c89f1efe09ba1b4 Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Tue, 1 Sep 2020 21:02:33 +0200 Subject: [PATCH 05/16] fix gc running too soon --- btv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/btv b/btv index da0ad2f..abd69bc 100755 --- a/btv +++ b/btv @@ -349,13 +349,13 @@ def do_create(args): print("!!! no previous Rank 2 snapshot, please create one using btv stream") snapshot.rank = 1 - ## garbage collection - do_gc() - ## save all snapshot metadata snapshot.dump() print(">>> Snapshot created: rank %d %s" %(snapshot.rank, snapshot.name)) + + ## garbage collection + do_gc() def do_list(args): """ From f58353fe87ec10f34581dbbc14d4ae23c1e5f462 Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Thu, 3 Sep 2020 23:30:49 +0200 Subject: [PATCH 06/16] Add --rank2 to btv snapshot --- btv | 53 +++++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/btv b/btv index abd69bc..cff65e4 100755 --- a/btv +++ b/btv @@ -278,30 +278,33 @@ def do_create(args): ## determine the rank of the new snapshot ## - snaps_since_rank_1 = 1 - snaps_since_rank_2 = 1 - - for snap in list_snapshots(): - if snap.rank == 1: - snaps_since_rank_1 = 1 - else: - snaps_since_rank_1 += 1 + if "--rank2" in args: + snapshot.rank = 2 + else: + snaps_since_rank_1 = 1 + snaps_since_rank_2 = 1 - if snap.rank == 2: - snaps_since_rank_1 = 1 - snaps_since_rank_2 = 1 - else: - snaps_since_rank_2 += 1 - - ## promote the snapshot - if snaps_since_rank_2 >= cfg.getint("snap", "rank_2_interval"): - if "--process" in args: - snapshot.rank = 2 - else: - print("!!! Rank 2 snapshot is due, please enable --process") + for snap in list_snapshots(): + if snap.rank == 1: + snaps_since_rank_1 = 1 + else: + snaps_since_rank_1 += 1 + + if snap.rank == 2: + snaps_since_rank_1 = 1 + snaps_since_rank_2 = 1 + else: + snaps_since_rank_2 += 1 + + ## promote the snapshot + if snaps_since_rank_2 >= cfg.getint("snap", "rank_2_interval"): + if "--process" in args: + snapshot.rank = 2 + else: + print("!!! Rank 2 snapshot is due, please enable --process") + snapshot.rank = 1 + elif snaps_since_rank_1 >= cfg.getint("snap", "rank_1_interval"): snapshot.rank = 1 - elif snaps_since_rank_1 >= cfg.getint("snap", "rank_1_interval"): - snapshot.rank = 1 ## create the snapshot itself ## @@ -540,8 +543,10 @@ def print_help(): {cmd} VERB [args] Verbs: - snapshot [--process] - Create a snapshot. If --process is passed, do all optional work. + snapshot [--process] [--rank2] + Create a snapshot. If --process is passed, do all optional work. If + --rank2 is passed, the new snapshot is automatically promoted to Rank 2 + and --process is implied. stream full SNAPSHOT OUTPUT_DIR Streams the SNAPSHOT into OUTPUT_DIR as a full (milestone) bin. From 5c57e266520cbb25877bcaea1fd126f3e930217c Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Tue, 24 Nov 2020 21:20:28 +0100 Subject: [PATCH 07/16] Add btv gc NUMBER command --- btv | 47 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/btv b/btv index cff65e4..85646c9 100755 --- a/btv +++ b/btv @@ -327,7 +327,6 @@ def do_create(args): ## do optional processing ## if snapshot.rank == 2: - ## snapshot serialization # if there's a previous Rank 2 snapshot, compute a diff against it # if not or if the process fails, demote this snap to Rank 1 @@ -493,23 +492,42 @@ def do_gc(args=None): Drops old snapshots. If the only arg is "greedy", drops ALL snapshots except the youngest - Rank 2. + Rank 2. If it's a number, drops that many oldest snapshots. """ - if args and "greedy" in args: - newest = list_snapshots(2)[-1] - - if newest: - print(">>> Dropping all snapshots except the newest Rank 2 in 5 s...") - time.sleep(5) + if args: + if args[0] == "greedy": + newest = list_snapshots(2)[-1] - for snap in list_snapshots(): - if snap.name != newest.name: - snap.drop() + if newest: + print(">>> Dropping all snapshots except the newest Rank 2 in 5 s...") + time.sleep(5) + + for snap in list_snapshots(): + if snap.name != newest.name: + snap.drop() + + else: + print("!!! no Rank 2 snapshot exists") + sys.exit(1) else: - print("!!! no Rank 2 snapshot exists") - sys.exit(1) + try: + count = int(args[0]) + except ValueError: + print("!!! %s is not a count of snapshots to delete" %(args[0])) + sys.exit(1) + + snaps = list_snapshots()[:count] + + for snap in snaps: + print(" %d %s %s" %(snap.rank, snap.name, ", ".join(snap.notes))) + + print(">>> These snapshots will be dropped in 5 s...") + time.sleep(5) + + for snap in snaps: + snap.drop() else: counts = [0, 0, 0] # Rank 0, 1, 2 @@ -565,6 +583,9 @@ Verbs: gc greedy Drop ALL snapshots except the newest Rank 2. + + gc COUNT + Drop COUNT oldest snapshots. """.format(cmd=sys.argv[0])) if __name__ == "__main__": From 00e3baf117a5f12fefc311fd6f273e6a4e4dc369 Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Thu, 18 Feb 2021 01:41:21 +0100 Subject: [PATCH 08/16] add telemetry --- btv | 16 ++++++++++++++++ cfg/config.ini | 7 +++++++ 2 files changed, 23 insertions(+) diff --git a/btv b/btv index 85646c9..1bf9113 100755 --- a/btv +++ b/btv @@ -10,6 +10,8 @@ import os import shlex import sys import time +import socket +import urllib.request CONFIG = "/etc/btv/config.ini" LOCKFILE = "/run/lock/btv/serialization.lock" @@ -259,6 +261,12 @@ def serialize(snap, outdir, key, snap_from=None): return 0 +def ping(url): + try: + urllib.request.urlopen(url, timeout=10) + except socket.error as e: + print("Ping failed: %s" %(e)) + # ------------------------------------------------------------------------------ # Verbs @@ -306,6 +314,10 @@ def do_create(args): elif snaps_since_rank_1 >= cfg.getint("snap", "rank_1_interval"): snapshot.rank = 1 + ping_url = cfg.get("monitoring", "rank2_start_url") + if snapshot.rank == 2 and ping_url: + ping(ping_url) + ## create the snapshot itself ## os.makedirs(snapshot.path) @@ -358,6 +370,10 @@ def do_create(args): ## garbage collection do_gc() + + ping_url = cfg.get("monitoring", "rank2_end_url") + if snapshot.rank == 2 and ping_url: + ping(ping_url) def do_list(args): """ diff --git a/cfg/config.ini b/cfg/config.ini index cb00390..f3dc5a9 100644 --- a/cfg/config.ini +++ b/cfg/config.ini @@ -35,3 +35,10 @@ dir = /mnt/pool/subvol/_backup rank_0_count=25 rank_1_count=36 rank_2_count=42 + +[monitoring] +# GETs this URL before starting a rank2 snapdhot +rank2_start_url = + +# GETs this URL after successfully completing a rank2 snapshot +rank2_end_url = From b96c163c8f06ec338185c9698e9fd66490fa8319 Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Sun, 30 May 2021 23:40:51 +0200 Subject: [PATCH 09/16] chmod rank2 files read-only for owner and no-access for everyone else --- btv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/btv b/btv index 1bf9113..c84665e 100755 --- a/btv +++ b/btv @@ -258,6 +258,9 @@ def serialize(snap, outdir, key, snap_from=None): for file in os.listdir(directory): path = os.path.join(directory, file) os.chown(path, outdir_stat.st_uid, outdir_stat.st_gid) + + if path.endswith(".aes") or path.endswith(".sha512"): + os.chmod(path, 0o100) return 0 From 7a51d286f29b5a6ec69e07f58afb9e2fc4add0b0 Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Sun, 30 May 2021 23:46:13 +0200 Subject: [PATCH 10/16] fix a stupid mistake from the previous commit --- btv | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/btv b/btv index c84665e..1de3c82 100755 --- a/btv +++ b/btv @@ -249,7 +249,7 @@ def serialize(snap, outdir, key, snap_from=None): ## add a self-check executable with open(os.path.join(directory, "check-integrity.sh"), "w") as f: f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n") - os.chmod(f.name, 0o555) + os.chmod(f.name, 0o500) ## fix permissions and ownership of created objects outdir_stat = os.stat(outdir) @@ -260,7 +260,9 @@ def serialize(snap, outdir, key, snap_from=None): os.chown(path, outdir_stat.st_uid, outdir_stat.st_gid) if path.endswith(".aes") or path.endswith(".sha512"): - os.chmod(path, 0o100) + os.chmod(path, 0o400) + + return 0 From 71886bee68b11ff0433713087ecb4bc343146b9d Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Sun, 30 May 2021 23:53:47 +0200 Subject: [PATCH 11/16] chmod rank2 dir as well --- btv | 1 + 1 file changed, 1 insertion(+) diff --git a/btv b/btv index 1de3c82..1f9a5c1 100755 --- a/btv +++ b/btv @@ -254,6 +254,7 @@ def serialize(snap, outdir, key, snap_from=None): ## fix permissions and ownership of created objects outdir_stat = os.stat(outdir) os.chown(directory, outdir_stat.st_uid, outdir_stat.st_gid) + os.chmod(directory, 0o500) for file in os.listdir(directory): path = os.path.join(directory, file) From 8345eb3217f00633f8d80c9869cae4a1e117c545 Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Mon, 31 May 2021 12:08:53 +0200 Subject: [PATCH 12/16] chmod dirs rwx instead of r-x so they can be moved --- btv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btv b/btv index 1f9a5c1..d2348b1 100755 --- a/btv +++ b/btv @@ -254,7 +254,7 @@ def serialize(snap, outdir, key, snap_from=None): ## fix permissions and ownership of created objects outdir_stat = os.stat(outdir) os.chown(directory, outdir_stat.st_uid, outdir_stat.st_gid) - os.chmod(directory, 0o500) + os.chmod(directory, 0o700) for file in os.listdir(directory): path = os.path.join(directory, file) From 831d9ae9a292df58cc240562675600e368eec1db Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Sun, 1 Aug 2021 09:36:19 +0200 Subject: [PATCH 13/16] add unpack script to every snapshot --- btv | 6 +++++- unpack.sh | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 unpack.sh diff --git a/btv b/btv index d2348b1..5646eff 100755 --- a/btv +++ b/btv @@ -8,6 +8,7 @@ import datetime import json import os import shlex +import shutil import sys import time import socket @@ -246,11 +247,14 @@ def serialize(snap, outdir, key, snap_from=None): ## final touches ## - ## add a self-check executable + ## add self-check and unpack executables with open(os.path.join(directory, "check-integrity.sh"), "w") as f: f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n") os.chmod(f.name, 0o500) + shutil.copy("/usr/share/btv/unpack.sh", "unpack.sh") + os.chmod("unpack.sh", 0o500) + ## fix permissions and ownership of created objects outdir_stat = os.stat(outdir) os.chown(directory, outdir_stat.st_uid, outdir_stat.st_gid) diff --git a/unpack.sh b/unpack.sh new file mode 100644 index 0000000..f78a8ba --- /dev/null +++ b/unpack.sh @@ -0,0 +1,24 @@ +#! /bin/zsh + +TIMESTAMP=($(basename "$(pwd)")) +OUTDIR="$1" +KEYFILE="$2" + +function die { + >&2 echo "$2" + exit $1 +} + +[[ "$0" != "./unpack.sh" ]] && die 1 "This can only be executed from the snapshot directory itself." +[[ ! -d "$OUTDIR" ]] && die 1 "The first argument must be a directory to unpack subvolumes into." +[[ ! -f "$KEYFILE" ]] && die 1 "The second argument must be a readable keyfile." +./check-integrity.sh || die 2 "This snapshot failed integrity checks." + +### end of checks + +for ARCHIVE in *btrfs.zst.aes +do + openssl enc -d -aes-256-cbc -pbkdf2 -salt -pass "file:$KEYFILE" < "$ARCHIVE" | zstd -d | btrfs receive "$OUTDIR" || die 3 "Failed to unpack subvolume." + SUBVOL_NAME=${ARCHIVE%%.btrfs.zst.aes} + mv "${OUTDIR}/${SUBVOL_NAME}" "${OUTDIR}/${SUBVOL_NAME}.${TIMESTAMP[1]}" || die 4 "Failed to rename subvolume." +done From 63906d328f7f2c2f99d98b7d56ceee4d7a127a4f Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Sun, 1 Aug 2021 09:56:32 +0200 Subject: [PATCH 14/16] fix unpack.sh directory --- btv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btv b/btv index 5646eff..85701d2 100755 --- a/btv +++ b/btv @@ -252,7 +252,7 @@ def serialize(snap, outdir, key, snap_from=None): f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n") os.chmod(f.name, 0o500) - shutil.copy("/usr/share/btv/unpack.sh", "unpack.sh") + shutil.copy("/usr/share/btv/unpack.sh", os.path.join(directory, "unpack.sh")) os.chmod("unpack.sh", 0o500) ## fix permissions and ownership of created objects From bbd568260aa6e5fd5ba91dea0032f55f1b523856 Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Sun, 1 Aug 2021 10:14:40 +0200 Subject: [PATCH 15/16] fix unpack.sh chmod --- btv | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/btv b/btv index 85701d2..fd3ea57 100755 --- a/btv +++ b/btv @@ -252,8 +252,9 @@ def serialize(snap, outdir, key, snap_from=None): f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n") os.chmod(f.name, 0o500) - shutil.copy("/usr/share/btv/unpack.sh", os.path.join(directory, "unpack.sh")) - os.chmod("unpack.sh", 0o500) + unpack_path = os.path.join(directory, "unpack.sh") + shutil.copy("/usr/share/btv/unpack.sh", unpack_path) + os.chmod(unpack_path, 0o500) ## fix permissions and ownership of created objects outdir_stat = os.stat(outdir) From 83bac5876dc98dc46f8b4c02c06f718d8521e45a Mon Sep 17 00:00:00 2001 From: Martin Sekera Date: Sun, 15 Jan 2023 18:53:52 +0100 Subject: [PATCH 16/16] simplify shapshot naming --- btv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/btv b/btv index fd3ea57..206c75c 100755 --- a/btv +++ b/btv @@ -199,9 +199,9 @@ def serialize(snap, outdir, key, snap_from=None): ## prepare directories ## if snap_from: - name = "%s diff from %s" %(snap.name, snap_from.name) + name = "%s to %s" %(snap_from.name, snap.name) else: - name = "%s full" %(snap.name) + name = snap.name directory = os.path.join(outdir, name) os.makedirs(directory)