btv 0.3.0
When serializing, create a lockfile which prevents the automatic snapshot service from starting. Add diff mode to the stream command.
This commit is contained in:
Martin Sekera
parent
35f0e38831
commit
cdac2e8ab4
2 changed files with 126 additions and 49 deletions
174
btv
174
btv
|
|
@ -11,6 +11,7 @@ import shlex
|
|||
import sys
|
||||
|
||||
CONFIG = "/etc/btv/config.ini"
|
||||
LOCKFILE = "/run/lock/btv/serialization.lock"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Global
|
||||
|
|
@ -33,6 +34,20 @@ def chdir(new_dir):
|
|||
finally:
|
||||
os.chdir(previous_dir)
|
||||
|
||||
@contextmanager
|
||||
def lockfile(path):
|
||||
d = os.path.dirname(path)
|
||||
|
||||
if not os.path.exists(d):
|
||||
os.makedirs(d)
|
||||
|
||||
f = open(path, "w")
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
os.remove(path)
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Functions
|
||||
|
||||
|
|
@ -172,6 +187,8 @@ def serialize(snap, outdir, key, snap_from=None):
|
|||
|
||||
Snap and snap_from are Snapshot objects.
|
||||
|
||||
A lockfile is kept for the duration of the process.
|
||||
|
||||
Returns 0 on success.
|
||||
"""
|
||||
|
||||
|
|
@ -196,39 +213,40 @@ def serialize(snap, outdir, key, snap_from=None):
|
|||
|
||||
return 1
|
||||
|
||||
## serialize each subvolume or subvolume pair
|
||||
##
|
||||
for subvolume in snap.subvolumes:
|
||||
if snap_from:
|
||||
btrfs_send = 'btrfs send -p "%s" "%s"' %(
|
||||
os.path.join(snap_from.path, subvolume),
|
||||
os.path.join(snap.path, subvolume)
|
||||
)
|
||||
else:
|
||||
btrfs_send = 'btrfs send "%s"' %(
|
||||
os.path.join(snap.path, subvolume)
|
||||
)
|
||||
with lockfile(LOCKFILE):
|
||||
## serialize each subvolume or subvolume pair
|
||||
##
|
||||
for subvolume in snap.subvolumes:
|
||||
if snap_from:
|
||||
btrfs_send = 'btrfs send -p "%s" "%s"' %(
|
||||
os.path.join(snap_from.path, subvolume),
|
||||
os.path.join(snap.path, subvolume)
|
||||
)
|
||||
else:
|
||||
btrfs_send = 'btrfs send "%s"' %(
|
||||
os.path.join(snap.path, subvolume)
|
||||
)
|
||||
|
||||
error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" > "%s.btrfs.zst.aes"' %(
|
||||
btrfs_send,
|
||||
cfg.get("crypto", "keyfile"),
|
||||
os.path.join(directory, subvolume)
|
||||
))
|
||||
|
||||
if error:
|
||||
print(" !! failed to serialize %s" %(subvolume))
|
||||
return error
|
||||
|
||||
error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" > "%s.btrfs.zst.aes"' %(
|
||||
btrfs_send,
|
||||
cfg.get("crypto", "keyfile"),
|
||||
os.path.join(directory, subvolume)
|
||||
## calculate checksums and add a self-check executable
|
||||
## FIXME calculate this on the fly, re-reading is expensive
|
||||
previous_wd = os.getcwd()
|
||||
os.chdir(directory)
|
||||
os.system('sha512sum "%s" > manifest.sha512' %(
|
||||
'" "'.join("%s.btrfs.zst.aes" %(s) for s in snap.subvolumes)
|
||||
))
|
||||
|
||||
if error:
|
||||
print(" !! failed to serialize %s" %(subvolume))
|
||||
return error
|
||||
|
||||
## calculate checksums and add a self-check executable
|
||||
## FIXME calculate this on the fly, re-reading is expensive
|
||||
previous_wd = os.getcwd()
|
||||
os.chdir(directory)
|
||||
os.system('sha512sum "%s" > manifest.sha512' %(
|
||||
'" "'.join("%s.btrfs.zst.aes" %(s) for s in snap.subvolumes)
|
||||
))
|
||||
|
||||
with open("check-integrity.sh", "w") as f:
|
||||
f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n")
|
||||
with open("check-integrity.sh", "w") as f:
|
||||
f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n")
|
||||
|
||||
## fix permissions and ownership of created objects
|
||||
os.chmod("check-integrity.sh", 0o555)
|
||||
|
|
@ -380,39 +398,93 @@ def do_drop(args):
|
|||
|
||||
def do_stream(args):
|
||||
"""
|
||||
Stream the snapshot args[0] into dir args[1].
|
||||
args are either
|
||||
|
||||
The affected snapshot is then promoted to Rank 2.
|
||||
"diff" SNAPSHOT_FROM SNAPSHOT_TO OUTPUT_DIR
|
||||
|
||||
or
|
||||
|
||||
"full" SNAPSHOT OUTPUT_DIR
|
||||
|
||||
SNAPSHOT_FROM must be Rank 2.
|
||||
|
||||
Streams the full or diff snapshot into OUTPUT_DIR.
|
||||
|
||||
The SNAPSHOT or SNAPSHOT_TO is then promoted to Rank 2.
|
||||
"""
|
||||
|
||||
snapshot = get_snapshot_by_name(args[0])
|
||||
## args interpretation and validation
|
||||
##
|
||||
if not args:
|
||||
raise UsageError("no args")
|
||||
|
||||
if not snapshot:
|
||||
print("!!! %s is not a snapshot" %(args[0]))
|
||||
if args[0] == "diff":
|
||||
if len(args) != 4:
|
||||
raise UsageError("'stream diff' requires exactly 3 arguments")
|
||||
|
||||
snap_from_name, snap_name, output_dir = args[1:]
|
||||
snap_from = get_snapshot_by_name(snap_from_name)
|
||||
|
||||
if not snap_from:
|
||||
print("!!! %s is not a snapshot" %(snap_from_name))
|
||||
sys.exit(2)
|
||||
|
||||
if snap_from.rank != 2:
|
||||
print("!!! source snapshot must be Rank 2, %s is %d" %(snap_from.name, snap_from.rank))
|
||||
print(' > Hint: btv stream full %s "%s"' %(snap_from.name, output_dir))
|
||||
sys.exit(3)
|
||||
|
||||
elif args[0] == "full":
|
||||
if len(args) != 3:
|
||||
raise UsageError("'stream full' requires exactly 2 arguments")
|
||||
|
||||
snap_name, output_dir = args[1:]
|
||||
snap_from = None
|
||||
|
||||
else:
|
||||
raise UsageError("'stream' type is either 'full' or 'diff'")
|
||||
|
||||
snap = get_snapshot_by_name(snap_name)
|
||||
|
||||
if not snap:
|
||||
print("!!! %s is not a snapshot" %(snap_name))
|
||||
sys.exit(2)
|
||||
|
||||
directory = args[1]
|
||||
|
||||
if not os.path.isdir(directory):
|
||||
print("!!! %s is not a directory" %(directory))
|
||||
if not os.path.isdir(output_dir):
|
||||
print("!!! %s is not a directory" %(output_dir))
|
||||
sys.exit(2)
|
||||
|
||||
print(">>> Serializing %s into %s" %(snapshot.name, directory))
|
||||
if snap_from and snap_from.name >= snap.name:
|
||||
print("!!! source snapshot is younger than target snapshot")
|
||||
sys.exit(3)
|
||||
|
||||
## serialization work
|
||||
##
|
||||
if snap_from:
|
||||
comment = "diff %s -> %s" %(snap_from.name, snap.name)
|
||||
else:
|
||||
comment = "full %s" %(snap.name)
|
||||
|
||||
print(">>> Serializing %s into %s" %(comment, output_dir))
|
||||
|
||||
error = serialize(
|
||||
snapshot,
|
||||
directory,
|
||||
cfg.get("crypto", "keyfile")
|
||||
snap,
|
||||
output_dir,
|
||||
cfg.get("crypto", "keyfile"),
|
||||
snap_from
|
||||
)
|
||||
|
||||
if error:
|
||||
print("!!! serialization failed")
|
||||
else:
|
||||
snapshot.rank = 2
|
||||
snapshot.notes.add("fully streamed")
|
||||
snapshot.dump()
|
||||
snap.rank = 2
|
||||
snap.notes.add("manually streamed")
|
||||
|
||||
print("<<< %s serialized and promoted to Rank %d" %(snapshot.name, snapshot.rank))
|
||||
if not snap_from:
|
||||
snap.notes.add("fully streamed")
|
||||
snap.dump()
|
||||
|
||||
print("<<< %s serialized and promoted to Rank 2" %(snap.name))
|
||||
|
||||
def do_gc(args=None):
|
||||
"""
|
||||
|
|
@ -453,9 +525,12 @@ Verbs:
|
|||
snapshot [--process]
|
||||
Create a snapshot. If --process is passed, do all optional work.
|
||||
|
||||
stream SNAPSHOT OUTPUT_DIR
|
||||
stream full SNAPSHOT OUTPUT_DIR
|
||||
Streams the SNAPSHOT into OUTPUT_DIR as a full (milestone) bin.
|
||||
|
||||
stream diff SNAPSHOT_FROM SNAPSHOT_TO OUTPUT_DIR
|
||||
Streams the difference between SNAPSHOTs into OUTPUT_DIR as a diff bin.
|
||||
|
||||
list
|
||||
List snapshots and their rank.
|
||||
|
||||
|
|
@ -480,7 +555,8 @@ if __name__ == "__main__":
|
|||
|
||||
try:
|
||||
verb_router[verb](sys.argv[2:])
|
||||
except UsageError:
|
||||
except UsageError as e:
|
||||
print("!!! %s" %(e))
|
||||
print()
|
||||
print_help()
|
||||
sys.exit(1)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
[Unit]
|
||||
Description=Filesystem snapshot (with offsite backup)
|
||||
ConditionPathExists=!/run/lock/btv/serialization.lock
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue