btv

@@ -8,11 +8,7 @@ import datetime
 import json
 import os
 import shlex
-import shutil
 import sys
-import time
-import socket
-import urllib.request
 
 CONFIG = "/etc/btv/config.ini"
 LOCKFILE = "/run/lock/btv/serialization.lock"
@@ -199,9 +195,9 @@ def serialize(snap, outdir, key, snap_from=None):
 	## prepare directories
 	##
 	if snap_from:
-		name = "%s to %s" %(snap_from.name, snap.name)
+		name = "%s diff from %s" %(snap.name, snap_from.name)
 	else:
-		name = snap.name
+		name = "%s full" %(snap.name)
 	
 	directory = os.path.join(outdir, name)
 	os.makedirs(directory)
@@ -217,10 +213,9 @@ def serialize(snap, outdir, key, snap_from=None):
 			
 			return 1
 	
-	## serialization (most expensive)
-	##
 	with lockfile(LOCKFILE):
-
+		## serialize each subvolume or subvolume pair
+		##
 		for subvolume in snap.subvolumes:
 			if snap_from:
 				btrfs_send = 'btrfs send -p "%s" "%s"' %(
@@ -232,11 +227,9 @@ def serialize(snap, outdir, key, snap_from=None):
 					os.path.join(snap.path, subvolume)
 				)
 			
-			error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" | hash-pipe sha512 "%s.btrfs.zst.aes" "%s/manifest.sha512" > "%s.btrfs.zst.aes"' %(
+			error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" > "%s.btrfs.zst.aes"' %(
 				btrfs_send,
 				cfg.get("crypto", "keyfile"),
-				subvolume,
-				directory,
 				os.path.join(directory, subvolume)
 			))
 			
@@ -244,40 +237,29 @@ def serialize(snap, outdir, key, snap_from=None):
 				print(" !! failed to serialize %s" %(subvolume))
 				return error
 		
-	## final touches
+		## calculate checksums and add a self-check executable
-	##
+		## FIXME calculate this on the fly, re-reading is expensive
+		previous_wd = os.getcwd()
+		os.chdir(directory)
+		os.system('sha512sum "%s" > manifest.sha512' %(
+			'" "'.join("%s.btrfs.zst.aes" %(s) for s in snap.subvolumes)
+		))
 		
-	## add self-check and unpack executables
+		with open("check-integrity.sh", "w") as f:
-	with open(os.path.join(directory, "check-integrity.sh"), "w") as f:
+			f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n")
-		f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n")
-		os.chmod(f.name, 0o500)
-	
-	unpack_path = os.path.join(directory, "unpack.sh")
-	shutil.copy("/usr/share/btv/unpack.sh", unpack_path)
-	os.chmod(unpack_path, 0o500)
 	
 	## fix permissions and ownership of created objects
+	os.chmod("check-integrity.sh", 0o555)
+	
 	outdir_stat = os.stat(outdir)
 	os.chown(directory, outdir_stat.st_uid, outdir_stat.st_gid)
-	os.chmod(directory, 0o700)
 	
 	for file in os.listdir(directory):
 		path = os.path.join(directory, file)
 		os.chown(path, outdir_stat.st_uid, outdir_stat.st_gid)
 	
-		if path.endswith(".aes") or path.endswith(".sha512"):
-			os.chmod(path, 0o400)
-		
-		
-	
 	return 0
 
-def ping(url):
-	try:
-		urllib.request.urlopen(url, timeout=10)
-	except socket.error as e:
-		print("Ping failed: %s" %(e))
-
 # ------------------------------------------------------------------------------
 # Verbs
 
@@ -297,37 +279,30 @@ def do_create(args):
 	
 	## determine the rank of the new snapshot
 	##
-	if "--rank2" in args:
+	snaps_since_rank_1 = 1
-		snapshot.rank = 2
+	snaps_since_rank_2 = 1
-	else:
-		snaps_since_rank_1 = 1
-		snaps_since_rank_2 = 1
 	
-		for snap in list_snapshots():
+	for snap in list_snapshots():
-			if snap.rank == 1:
+		if snap.rank == 1:
-				snaps_since_rank_1 = 1
+			snaps_since_rank_1 = 1
-			else:
+		else:
-				snaps_since_rank_1 += 1
+			snaps_since_rank_1 += 1
 		
-			if snap.rank == 2:
+		if snap.rank == 2:
-				snaps_since_rank_1 = 1
+			snaps_since_rank_1 = 1
-				snaps_since_rank_2 = 1
+			snaps_since_rank_2 = 1
-			else:
+		else:
-				snaps_since_rank_2 += 1
+			snaps_since_rank_2 += 1
 	
-		## promote the snapshot
+	## promote the snapshot
-		if snaps_since_rank_2 >= cfg.getint("snap", "rank_2_interval"):
+	if snaps_since_rank_2 >= cfg.getint("snap", "rank_2_interval"):
-			if "--process" in args:
+		if "--process" in args:
-				snapshot.rank = 2
+			snapshot.rank = 2
-			else:
+		else:
-				print("!!! Rank 2 snapshot is due, please enable --process")
+			print("!!! Rank 2 snapshot is due, please enable --process")
-				snapshot.rank = 1
-		elif snaps_since_rank_1 >= cfg.getint("snap", "rank_1_interval"):
 			snapshot.rank = 1
-	
+	elif snaps_since_rank_1 >= cfg.getint("snap", "rank_1_interval"):
-	ping_url = cfg.get("monitoring", "rank2_start_url")
+		snapshot.rank = 1
-	if snapshot.rank == 2 and ping_url:
-		ping(ping_url)
 	
 	## create the snapshot itself
 	##
@@ -350,6 +325,7 @@ def do_create(args):
 	## do optional processing
 	##
 	if snapshot.rank == 2:
+		
 		## snapshot serialization
 		# if there's a previous Rank 2 snapshot, compute a diff against it
 		# if not or if the process fails, demote this snap to Rank 1
@@ -374,18 +350,14 @@ def do_create(args):
 			print("!!! no previous Rank 2 snapshot, please create one using btv stream")
 			snapshot.rank = 1
 		
+		## garbage collection
+		do_gc()
+	
 	## save all snapshot metadata
 	snapshot.dump()
 	
 	print(">>> Snapshot created: rank %d %s" %(snapshot.rank, snapshot.name))
 
-	## garbage collection
-	do_gc()
-	
-	ping_url = cfg.get("monitoring", "rank2_end_url")
-	if snapshot.rank == 2 and ping_url:
-		ping(ping_url)
-
 def do_list(args):
 	"""
 	Print a list of existing snapshots.
@@ -517,60 +489,22 @@ def do_stream(args):
 def do_gc(args=None):
 	"""
 	Drops old snapshots.
-	
-	If the only arg is "greedy", drops ALL snapshots except the youngest
-	Rank 2. If it's a number, drops that many oldest snapshots.
 	"""
 	
-	if args: 
+	counts = [0, 0, 0] # Rank 0, 1, 2
-		if args[0] == "greedy":
+	limits = [
-			newest = list_snapshots(2)[-1]
+		cfg.getint("gc", "rank_0_count"),
+		cfg.getint("gc", "rank_1_count"),
+		cfg.getint("gc", "rank_2_count")
+	]
 	
-			if newest:
+	for snap in reversed(list_snapshots()):
-				print(">>> Dropping all snapshots except the newest Rank 2 in 5 s...")
+		counts[snap.rank] += 1
-				time.sleep(5)
 		
-				for snap in list_snapshots():
+		if counts[snap.rank] > limits[snap.rank]:
-					if snap.name != newest.name:
+			print(" >> delete Rank %d %s" %(snap.rank, snap.name))
-						snap.drop()
 			
-			else:
+			snap.drop()
-				print("!!! no Rank 2 snapshot exists")
-				sys.exit(1)
-		
-		else:
-			try:
-				count = int(args[0])
-			except ValueError:
-				print("!!! %s is not a count of snapshots to delete" %(args[0]))
-				sys.exit(1)
-			
-			snaps = list_snapshots()[:count]
-			
-			for snap in snaps:
-				print("   %d %s %s" %(snap.rank, snap.name, ", ".join(snap.notes)))
-			
-			print(">>> These snapshots will be dropped in 5 s...")
-			time.sleep(5)
-			
-			for snap in snaps:
-				snap.drop()
-	
-	else:
-		counts = [0, 0, 0] # Rank 0, 1, 2
-		limits = [
-			cfg.getint("gc", "rank_0_count"),
-			cfg.getint("gc", "rank_1_count"),
-			cfg.getint("gc", "rank_2_count")
-		]
-		
-		for snap in reversed(list_snapshots()):
-			counts[snap.rank] += 1
-			
-			if counts[snap.rank] > limits[snap.rank]:
-				print(" >> delete Rank %d %s" %(snap.rank, snap.name))
-				
-				snap.drop()
 
 verb_router = {
 	"snapshot": do_create,
@@ -588,10 +522,8 @@ def print_help():
   {cmd} VERB [args]
 
 Verbs:
-  snapshot [--process] [--rank2]
+  snapshot [--process]
-    Create a snapshot. If --process is passed, do all optional work. If
+    Create a snapshot. If --process is passed, do all optional work.
-    --rank2 is passed, the new snapshot is automatically promoted to Rank 2
-    and --process is implied.
   
   stream full SNAPSHOT OUTPUT_DIR
     Streams the SNAPSHOT into OUTPUT_DIR as a full (milestone) bin.
@@ -607,12 +539,6 @@ Verbs:
   
   gc
     Drops old local snapshots based on garbage collector settings.
-  
-  gc greedy
-    Drop ALL snapshots except the newest Rank 2.
-  
-  gc COUNT
-    Drop COUNT oldest snapshots.
 """.format(cmd=sys.argv[0]))
 
 if __name__ == "__main__":

cfg/config.ini

@@ -30,15 +30,9 @@ dir = /mnt/pool/subvol/_backup
 
 [gc]
 # How many snapshots of each Rank to keep around.
+# Note: pruning is only performed when a new Rank 2 snapshot is attempted.
 # Default values are 25, 36, and 42, i.e. 5 hours for Rank 0, 2 days for Rank 1,
 # and a week for Rank 2.
 rank_0_count=25
 rank_1_count=36
 rank_2_count=42
-
-[monitoring]
-# GETs this URL before starting a rank2 snapdhot
-rank2_start_url = 
-
-# GETs this URL after successfully completing a rank2 snapshot
-rank2_end_url = 

unpack.sh

@@ -1,24 +0,0 @@
-#! /bin/zsh
-
-TIMESTAMP=($(basename "$(pwd)"))
-OUTDIR="$1"
-KEYFILE="$2"
-
-function die {
-	>&2 echo "$2"
-	exit $1
-}
-
-[[ "$0" != "./unpack.sh" ]] && die 1 "This can only be executed from the snapshot directory itself."
-[[ ! -d "$OUTDIR" ]]       && die 1 "The first argument must be a directory to unpack subvolumes into."
-[[ ! -f "$KEYFILE" ]]      && die 1 "The second argument must be a readable keyfile."
-./check-integrity.sh       || die 2 "This snapshot failed integrity checks."
-
-### end of checks
-
-for ARCHIVE in *btrfs.zst.aes
-do
-	openssl enc -d -aes-256-cbc -pbkdf2 -salt -pass "file:$KEYFILE" < "$ARCHIVE" | zstd -d | btrfs receive "$OUTDIR" || die 3 "Failed to unpack subvolume."
-	SUBVOL_NAME=${ARCHIVE%%.btrfs.zst.aes}
-	mv "${OUTDIR}/${SUBVOL_NAME}" "${OUTDIR}/${SUBVOL_NAME}.${TIMESTAMP[1]}" || die 4 "Failed to rename subvolume."
-done