btv 0.3.0

When serializing, create a lockfile which prevents the automatic
snapshot service from starting.

Add diff mode to the stream command.

btv

@@ -11,6 +11,7 @@ import shlex
 import sys
 
 CONFIG = "/etc/btv/config.ini"
+LOCKFILE = "/run/lock/btv/serialization.lock"
 
 # ------------------------------------------------------------------------------
 # Global
@@ -33,6 +34,20 @@ def chdir(new_dir):
 	finally:
 		os.chdir(previous_dir)
 
+@contextmanager
+def lockfile(path):
+	d = os.path.dirname(path)
+	
+	if not os.path.exists(d):
+		os.makedirs(d)
+	
+	f = open(path, "w")
+	
+	try:
+		yield
+	finally:
+		os.remove(path)
+
 # ------------------------------------------------------------------------------
 # Functions
 
@@ -172,6 +187,8 @@ def serialize(snap, outdir, key, snap_from=None):
 	
 	Snap and snap_from are Snapshot objects.
 	
+	A lockfile is kept for the duration of the process.
+	
 	Returns 0 on success.
 	"""
 	
@@ -196,39 +213,40 @@ def serialize(snap, outdir, key, snap_from=None):
 			
 			return 1
 	
-	## serialize each subvolume or subvolume pair
+	with lockfile(LOCKFILE):
-	##
+		## serialize each subvolume or subvolume pair
-	for subvolume in snap.subvolumes:
+		##
-		if snap_from:
+		for subvolume in snap.subvolumes:
-			btrfs_send = 'btrfs send -p "%s" "%s"' %(
+			if snap_from:
-				os.path.join(snap_from.path, subvolume),
+				btrfs_send = 'btrfs send -p "%s" "%s"' %(
-				os.path.join(snap.path, subvolume)
+					os.path.join(snap_from.path, subvolume),
-			)
+					os.path.join(snap.path, subvolume)
-		else:
+				)
-			btrfs_send = 'btrfs send "%s"' %(
+			else:
-				os.path.join(snap.path, subvolume)
+				btrfs_send = 'btrfs send "%s"' %(
-			)
+					os.path.join(snap.path, subvolume)
+				)
+			
+			error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" > "%s.btrfs.zst.aes"' %(
+				btrfs_send,
+				cfg.get("crypto", "keyfile"),
+				os.path.join(directory, subvolume)
+			))
+			
+			if error:
+				print(" !! failed to serialize %s" %(subvolume))
+				return error
 		
-		error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" > "%s.btrfs.zst.aes"' %(
+		## calculate checksums and add a self-check executable
-			btrfs_send,
+		## FIXME calculate this on the fly, re-reading is expensive
-			cfg.get("crypto", "keyfile"),
+		previous_wd = os.getcwd()
-			os.path.join(directory, subvolume)
+		os.chdir(directory)
+		os.system('sha512sum "%s" > manifest.sha512' %(
+			'" "'.join("%s.btrfs.zst.aes" %(s) for s in snap.subvolumes)
 		))
 		
-		if error:
+		with open("check-integrity.sh", "w") as f:
-			print(" !! failed to serialize %s" %(subvolume))
+			f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n")
-			return error
-	
-	## calculate checksums and add a self-check executable
-	## FIXME calculate this on the fly, re-reading is expensive
-	previous_wd = os.getcwd()
-	os.chdir(directory)
-	os.system('sha512sum "%s" > manifest.sha512' %(
-		'" "'.join("%s.btrfs.zst.aes" %(s) for s in snap.subvolumes)
-	))
-	
-	with open("check-integrity.sh", "w") as f:
-		f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n")
 	
 	## fix permissions and ownership of created objects
 	os.chmod("check-integrity.sh", 0o555)
@@ -380,39 +398,93 @@ def do_drop(args):
 
 def do_stream(args):
 	"""
-	Stream the snapshot args[0] into dir args[1].
+	args are either
 	
-	The affected snapshot is then promoted to Rank 2.
+	  "diff" SNAPSHOT_FROM SNAPSHOT_TO OUTPUT_DIR
+	
+	or
+	
+	  "full" SNAPSHOT OUTPUT_DIR
+	
+	SNAPSHOT_FROM must be Rank 2.
+	
+	Streams the full or diff snapshot into OUTPUT_DIR.
+	
+	The SNAPSHOT or SNAPSHOT_TO is then promoted to Rank 2.
 	"""
 	
-	snapshot = get_snapshot_by_name(args[0])
+	## args interpretation and validation
+	##
+	if not args:
+		raise UsageError("no args")
 	
-	if not snapshot:
+	if args[0] == "diff":
-		print("!!! %s is not a snapshot" %(args[0]))
+		if len(args) != 4:
+			raise UsageError("'stream diff' requires exactly 3 arguments")
+		
+		snap_from_name, snap_name, output_dir = args[1:]
+		snap_from = get_snapshot_by_name(snap_from_name)
+		
+		if not snap_from:
+			print("!!! %s is not a snapshot" %(snap_from_name))
+			sys.exit(2)
+		
+		if snap_from.rank != 2:
+			print("!!! source snapshot must be Rank 2, %s is %d" %(snap_from.name, snap_from.rank))
+			print('  > Hint: btv stream full %s "%s"' %(snap_from.name, output_dir))
+			sys.exit(3)
+	
+	elif args[0] == "full":
+		if len(args) != 3:
+			raise UsageError("'stream full' requires exactly 2 arguments")
+		
+		snap_name, output_dir = args[1:]
+		snap_from = None
+	
+	else:
+		raise UsageError("'stream' type is either 'full' or 'diff'")
+	
+	snap = get_snapshot_by_name(snap_name)
+	
+	if not snap:
+		print("!!! %s is not a snapshot" %(snap_name))
 		sys.exit(2)
 	
-	directory = args[1]
+	if not os.path.isdir(output_dir):
-	
+		print("!!! %s is not a directory" %(output_dir))
-	if not os.path.isdir(directory):
-		print("!!! %s is not a directory" %(directory))
 		sys.exit(2)
 	
-	print(">>> Serializing %s into %s" %(snapshot.name, directory))
+	if snap_from and snap_from.name >= snap.name:
+		print("!!! source snapshot is younger than target snapshot")
+		sys.exit(3)
+	
+	## serialization work
+	##
+	if snap_from:
+		comment = "diff %s -> %s" %(snap_from.name, snap.name)
+	else:
+		comment = "full %s" %(snap.name)
+	
+	print(">>> Serializing %s into %s" %(comment, output_dir))
 	
 	error = serialize(
-		snapshot,
+		snap,
-		directory,
+		output_dir,
-		cfg.get("crypto", "keyfile")
+		cfg.get("crypto", "keyfile"),
+		snap_from
 	)
 	
 	if error:
 		print("!!! serialization failed")
 	else:
-		snapshot.rank = 2
+		snap.rank = 2
-		snapshot.notes.add("fully streamed")
+		snap.notes.add("manually streamed")
-		snapshot.dump()
 		
-		print("<<< %s serialized and promoted to Rank %d" %(snapshot.name, snapshot.rank))
+		if not snap_from:
+			snap.notes.add("fully streamed")
+		snap.dump()
+		
+		print("<<< %s serialized and promoted to Rank 2" %(snap.name))
 
 def do_gc(args=None):
 	"""
@@ -453,9 +525,12 @@ Verbs:
   snapshot [--process]
     Create a snapshot. If --process is passed, do all optional work.
   
-  stream SNAPSHOT OUTPUT_DIR
+  stream full SNAPSHOT OUTPUT_DIR
     Streams the SNAPSHOT into OUTPUT_DIR as a full (milestone) bin.
   
+  stream diff SNAPSHOT_FROM SNAPSHOT_TO OUTPUT_DIR
+    Streams the difference between SNAPSHOTs into OUTPUT_DIR as a diff bin.
+  
   list
     List snapshots and their rank.
   
@@ -480,7 +555,8 @@ if __name__ == "__main__":
 	
 	try:
 		verb_router[verb](sys.argv[2:])
-	except UsageError:
+	except UsageError as e:
+		print("!!! %s" %(e))
 		print()
 		print_help()
 		sys.exit(1)

systemd/btv-backup.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Filesystem snapshot (with offsite backup)
+ConditionPathExists=!/run/lock/btv/serialization.lock
 
 [Service]
 Type=oneshot