diff --git a/btv b/btv index d75727f..113b9b4 100755 --- a/btv +++ b/btv @@ -11,6 +11,7 @@ import shlex import sys CONFIG = "/etc/btv/config.ini" +LOCKFILE = "/run/lock/btv/serialization.lock" # ------------------------------------------------------------------------------ # Global @@ -33,6 +34,20 @@ def chdir(new_dir): finally: os.chdir(previous_dir) +@contextmanager +def lockfile(path): + d = os.path.dirname(path) + + if not os.path.exists(d): + os.makedirs(d) + + f = open(path, "w") + + try: + yield + finally: + os.remove(path) + # ------------------------------------------------------------------------------ # Functions @@ -172,6 +187,8 @@ def serialize(snap, outdir, key, snap_from=None): Snap and snap_from are Snapshot objects. + A lockfile is kept for the duration of the process. + Returns 0 on success. """ @@ -196,39 +213,40 @@ def serialize(snap, outdir, key, snap_from=None): return 1 - ## serialize each subvolume or subvolume pair - ## - for subvolume in snap.subvolumes: - if snap_from: - btrfs_send = 'btrfs send -p "%s" "%s"' %( - os.path.join(snap_from.path, subvolume), - os.path.join(snap.path, subvolume) - ) - else: - btrfs_send = 'btrfs send "%s"' %( - os.path.join(snap.path, subvolume) - ) + with lockfile(LOCKFILE): + ## serialize each subvolume or subvolume pair + ## + for subvolume in snap.subvolumes: + if snap_from: + btrfs_send = 'btrfs send -p "%s" "%s"' %( + os.path.join(snap_from.path, subvolume), + os.path.join(snap.path, subvolume) + ) + else: + btrfs_send = 'btrfs send "%s"' %( + os.path.join(snap.path, subvolume) + ) + + error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" > "%s.btrfs.zst.aes"' %( + btrfs_send, + cfg.get("crypto", "keyfile"), + os.path.join(directory, subvolume) + )) + + if error: + print(" !! failed to serialize %s" %(subvolume)) + return error - error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" > "%s.btrfs.zst.aes"' %( - btrfs_send, - cfg.get("crypto", "keyfile"), - os.path.join(directory, subvolume) + ## calculate checksums and add a self-check executable + ## FIXME calculate this on the fly, re-reading is expensive + previous_wd = os.getcwd() + os.chdir(directory) + os.system('sha512sum "%s" > manifest.sha512' %( + '" "'.join("%s.btrfs.zst.aes" %(s) for s in snap.subvolumes) )) - if error: - print(" !! failed to serialize %s" %(subvolume)) - return error - - ## calculate checksums and add a self-check executable - ## FIXME calculate this on the fly, re-reading is expensive - previous_wd = os.getcwd() - os.chdir(directory) - os.system('sha512sum "%s" > manifest.sha512' %( - '" "'.join("%s.btrfs.zst.aes" %(s) for s in snap.subvolumes) - )) - - with open("check-integrity.sh", "w") as f: - f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n") + with open("check-integrity.sh", "w") as f: + f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n") ## fix permissions and ownership of created objects os.chmod("check-integrity.sh", 0o555) @@ -380,39 +398,93 @@ def do_drop(args): def do_stream(args): """ - Stream the snapshot args[0] into dir args[1]. + args are either - The affected snapshot is then promoted to Rank 2. + "diff" SNAPSHOT_FROM SNAPSHOT_TO OUTPUT_DIR + + or + + "full" SNAPSHOT OUTPUT_DIR + + SNAPSHOT_FROM must be Rank 2. + + Streams the full or diff snapshot into OUTPUT_DIR. + + The SNAPSHOT or SNAPSHOT_TO is then promoted to Rank 2. """ - snapshot = get_snapshot_by_name(args[0]) + ## args interpretation and validation + ## + if not args: + raise UsageError("no args") - if not snapshot: - print("!!! %s is not a snapshot" %(args[0])) + if args[0] == "diff": + if len(args) != 4: + raise UsageError("'stream diff' requires exactly 3 arguments") + + snap_from_name, snap_name, output_dir = args[1:] + snap_from = get_snapshot_by_name(snap_from_name) + + if not snap_from: + print("!!! %s is not a snapshot" %(snap_from_name)) + sys.exit(2) + + if snap_from.rank != 2: + print("!!! source snapshot must be Rank 2, %s is %d" %(snap_from.name, snap_from.rank)) + print(' > Hint: btv stream full %s "%s"' %(snap_from.name, output_dir)) + sys.exit(3) + + elif args[0] == "full": + if len(args) != 3: + raise UsageError("'stream full' requires exactly 2 arguments") + + snap_name, output_dir = args[1:] + snap_from = None + + else: + raise UsageError("'stream' type is either 'full' or 'diff'") + + snap = get_snapshot_by_name(snap_name) + + if not snap: + print("!!! %s is not a snapshot" %(snap_name)) sys.exit(2) - directory = args[1] - - if not os.path.isdir(directory): - print("!!! %s is not a directory" %(directory)) + if not os.path.isdir(output_dir): + print("!!! %s is not a directory" %(output_dir)) sys.exit(2) - print(">>> Serializing %s into %s" %(snapshot.name, directory)) + if snap_from and snap_from.name >= snap.name: + print("!!! source snapshot is younger than target snapshot") + sys.exit(3) + + ## serialization work + ## + if snap_from: + comment = "diff %s -> %s" %(snap_from.name, snap.name) + else: + comment = "full %s" %(snap.name) + + print(">>> Serializing %s into %s" %(comment, output_dir)) error = serialize( - snapshot, - directory, - cfg.get("crypto", "keyfile") + snap, + output_dir, + cfg.get("crypto", "keyfile"), + snap_from ) if error: print("!!! serialization failed") else: - snapshot.rank = 2 - snapshot.notes.add("fully streamed") - snapshot.dump() + snap.rank = 2 + snap.notes.add("manually streamed") - print("<<< %s serialized and promoted to Rank %d" %(snapshot.name, snapshot.rank)) + if not snap_from: + snap.notes.add("fully streamed") + snap.dump() + + print("<<< %s serialized and promoted to Rank 2" %(snap.name)) def do_gc(args=None): """ @@ -453,9 +525,12 @@ Verbs: snapshot [--process] Create a snapshot. If --process is passed, do all optional work. - stream SNAPSHOT OUTPUT_DIR + stream full SNAPSHOT OUTPUT_DIR Streams the SNAPSHOT into OUTPUT_DIR as a full (milestone) bin. + stream diff SNAPSHOT_FROM SNAPSHOT_TO OUTPUT_DIR + Streams the difference between SNAPSHOTs into OUTPUT_DIR as a diff bin. + list List snapshots and their rank. @@ -480,7 +555,8 @@ if __name__ == "__main__": try: verb_router[verb](sys.argv[2:]) - except UsageError: + except UsageError as e: + print("!!! %s" %(e)) print() print_help() sys.exit(1) diff --git a/systemd/btv-backup.service b/systemd/btv-backup.service index 19eb0cb..d63bec8 100644 --- a/systemd/btv-backup.service +++ b/systemd/btv-backup.service @@ -1,5 +1,6 @@ [Unit] Description=Filesystem snapshot (with offsite backup) +ConditionPathExists=!/run/lock/btv/serialization.lock [Service] Type=oneshot