full feature set implemented
This commit is contained in:
Martin Sekera
parent
8ce4f997df
commit
3e790cc2f3
2 changed files with 292 additions and 65 deletions
343
btv
343
btv
|
|
@ -42,13 +42,16 @@ def verify_keyfile():
|
|||
wrong set of permissions.
|
||||
"""
|
||||
|
||||
f = cfg.get("main", "keyfile")
|
||||
f = cfg.get("crypto", "keyfile")
|
||||
|
||||
if not os.path.exists(f):
|
||||
raise RuntimeError("does not exist")
|
||||
raise RuntimeError("keyfile does not exist")
|
||||
|
||||
s = os.stat(f)
|
||||
|
||||
if s.st_size < 5:
|
||||
raise RuntimeError("password in the keyfile is too short")
|
||||
|
||||
if s.st_uid != 0:
|
||||
raise RuntimeError("keyfile is not owned by root")
|
||||
|
||||
|
|
@ -68,7 +71,64 @@ class Snapshot:
|
|||
name: str
|
||||
rank: int
|
||||
|
||||
def get_snapshot(name):
|
||||
def __post_init__(self):
|
||||
self.path = os.path.join(cfg.get("snap", "dir"), self.name)
|
||||
self.subvolumes = set()
|
||||
self.notes = set()
|
||||
|
||||
@classmethod
|
||||
def load(cls, path):
|
||||
with open(os.path.join(path, ".meta")) as f:
|
||||
meta = json.load(f)
|
||||
snap = cls(meta["name"], meta["rank"])
|
||||
snap.subvolumes.update(meta["subvolumes"])
|
||||
snap.notes.update(meta["notes"])
|
||||
snap.path = path
|
||||
snap.check()
|
||||
|
||||
return snap
|
||||
|
||||
def dump(self):
|
||||
if not os.path.exists(self.path):
|
||||
os.makedirs(self.path)
|
||||
|
||||
with open(os.path.join(self.path, ".meta"), "w") as f:
|
||||
meta = {
|
||||
"name": self.name,
|
||||
"rank": self.rank,
|
||||
"subvolumes": sorted(self.subvolumes),
|
||||
"notes": sorted(self.notes)
|
||||
}
|
||||
|
||||
json.dump(meta, f)
|
||||
|
||||
def check(self):
|
||||
## verify all declared subvolumes exist
|
||||
contents = set(os.listdir(self.path))
|
||||
|
||||
if self.subvolumes - contents:
|
||||
self.notes.add("missing subvolumes")
|
||||
|
||||
def drop(self):
|
||||
for subvolume in self.subvolumes:
|
||||
btrfs_snap = os.path.join(self.path, subvolume)
|
||||
|
||||
if not os.path.exists(btrfs_snap):
|
||||
print(" !! subvolume not found: %s" %(btrfs_snap))
|
||||
return 1
|
||||
|
||||
error = os.system(shlex.join(("btrfs", "subvolume", "delete", btrfs_snap)))
|
||||
|
||||
if error:
|
||||
print(" !! error %d" %(error))
|
||||
return error
|
||||
|
||||
os.remove(os.path.join(self.path, ".meta"))
|
||||
os.rmdir(self.path)
|
||||
|
||||
return 0
|
||||
|
||||
def get_snapshot_by_name(name):
|
||||
"""
|
||||
Returns a Snapshot with the requested name iff such exists. Otherwise,
|
||||
returns None.
|
||||
|
|
@ -77,16 +137,11 @@ def get_snapshot(name):
|
|||
rootdir = cfg.get("snap", "dir")
|
||||
|
||||
try:
|
||||
with open(os.path.join(rootdir, name, ".meta")) as f:
|
||||
meta = json.load(f)
|
||||
return Snapshot(
|
||||
name,
|
||||
meta["rank"]
|
||||
)
|
||||
return Snapshot.load(os.path.join(rootdir, name))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
def list_snapshots():
|
||||
def list_snapshots(min_rank=0):
|
||||
"""
|
||||
Returns a list of all existing Snapshots sorted oldest to newest.
|
||||
"""
|
||||
|
|
@ -94,15 +149,95 @@ def list_snapshots():
|
|||
snaps = []
|
||||
rootdir = cfg.get("snap", "dir")
|
||||
|
||||
for snapdir in os.listdir(rootdir):
|
||||
snaps.append(get_snapshot(snapdir))
|
||||
for name in os.listdir(rootdir):
|
||||
s = get_snapshot_by_name(name)
|
||||
|
||||
if s and s.rank >= min_rank:
|
||||
snaps.append(s)
|
||||
|
||||
return snaps
|
||||
|
||||
def delete_snapshot(snap):
|
||||
"""
|
||||
Drops a snapshot referenced by the object.
|
||||
"""
|
||||
|
||||
|
||||
def serialize(snap, outdir, key, snap_from=None):
|
||||
"""
|
||||
Serializes a snapshot (differentially if snap_from is passed, fully
|
||||
otherwise) into a new directory under outdir.
|
||||
|
||||
Key is the path to an openssl-style passphrase file.
|
||||
|
||||
Snap and snap_from are Snapshot objects.
|
||||
|
||||
Returns 0 on success.
|
||||
"""
|
||||
|
||||
## prepare directories
|
||||
##
|
||||
if snap_from:
|
||||
name = "%s diff from %s" %(snap.name, snap_from.name)
|
||||
else:
|
||||
name = "%s full" %(snap.name)
|
||||
|
||||
directory = os.path.join(outdir, name)
|
||||
os.makedirs(directory)
|
||||
|
||||
## preflight checks
|
||||
##
|
||||
if snap_from:
|
||||
# I don't want to handle (dis)appearing subvolumes
|
||||
if not snap.subvolumes == snap_from.subvolumes:
|
||||
print(" !! snapshots have different subvolumes; please do a full stream")
|
||||
print(" %s: %s" %(snap.name, " ".join(snap.subvolumes)))
|
||||
print(" %s: %s" %(snap_from.name, " ".join(snap_from.subvolumes)))
|
||||
|
||||
return 1
|
||||
|
||||
## serialize each subvolume or subvolume pair
|
||||
##
|
||||
for subvolume in snap.subvolumes:
|
||||
if snap_from:
|
||||
btrfs_send = 'btrfs send -p "%s" "%s"' %(
|
||||
os.path.join(snap_from.path, subvolume),
|
||||
os.path.join(snap.path, subvolume)
|
||||
)
|
||||
else:
|
||||
btrfs_send = 'btrfs send "%s"' %(
|
||||
os.path.join(snap.path, subvolume)
|
||||
)
|
||||
|
||||
error = os.system('%s | zstd | openssl enc -e -aes-256-cbc -pbkdf2 -salt -pass "file:%s" > "%s.btrfs.zst.aes"' %(
|
||||
btrfs_send,
|
||||
cfg.get("crypto", "keyfile"),
|
||||
os.path.join(directory, subvolume)
|
||||
))
|
||||
|
||||
if error:
|
||||
print(" !! failed to serialize %s" %(subvolume))
|
||||
return error
|
||||
|
||||
## calculate checksums and add a self-check executable
|
||||
## FIXME calculate this on the fly, re-reading is expensive
|
||||
previous_wd = os.getcwd()
|
||||
os.chdir(directory)
|
||||
os.system('sha512sum "%s" > manifest.sha512' %(
|
||||
'" "'.join("%s.btrfs.zst.aes" %(s) for s in snap.subvolumes)
|
||||
))
|
||||
|
||||
with open("check-integrity.sh", "w") as f:
|
||||
f.write("#! /bin/sh\n\nsha512sum --check manifest.sha512\n")
|
||||
|
||||
os.chmod("check-integrity.sh", 0o500)
|
||||
|
||||
return 0
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Verbs
|
||||
|
||||
def do_create(args, context):
|
||||
def do_create(args):
|
||||
"""
|
||||
Bulk of the work happens here.
|
||||
|
||||
|
|
@ -111,17 +246,17 @@ def do_create(args, context):
|
|||
- calls gc if needed
|
||||
"""
|
||||
|
||||
snap_path = os.path.join(
|
||||
cfg.get("snap", "dir"),
|
||||
datetime.datetime.now().strftime("%Y-%m-%d.%H%M%S")
|
||||
snapshot = Snapshot(
|
||||
datetime.datetime.now().strftime("%Y-%m-%d.%H%M%S"),
|
||||
0
|
||||
)
|
||||
|
||||
# determine the rank of the new snapshot
|
||||
snaps = list_snapshots()
|
||||
## determine the rank of the new snapshot
|
||||
##
|
||||
snaps_since_rank_1 = 1
|
||||
snaps_since_rank_2 = 1
|
||||
|
||||
for snap in snaps:
|
||||
for snap in list_snapshots():
|
||||
if snap.rank == 1:
|
||||
snaps_since_rank_1 = 1
|
||||
else:
|
||||
|
|
@ -133,76 +268,158 @@ def do_create(args, context):
|
|||
else:
|
||||
snaps_since_rank_2 += 1
|
||||
|
||||
## promote the snapshot
|
||||
if snaps_since_rank_2 >= cfg.getint("snap", "rank_2_interval"):
|
||||
rank = 2
|
||||
elif snaps_since_rank_1 >= cfg.getint("snap", "rank_1_interval"):
|
||||
rank = 1
|
||||
if "--process" in args:
|
||||
snapshot.rank = 2
|
||||
else:
|
||||
rank = 0
|
||||
|
||||
print(">>> Creating Rank %d snapshot" %(rank), snap_path)
|
||||
|
||||
os.makedirs(snap_path)
|
||||
print("!!! Rank 2 snapshot is due, please enable --process")
|
||||
snapshot.rank = 1
|
||||
elif snaps_since_rank_1 >= cfg.getint("snap", "rank_1_interval"):
|
||||
snapshot.rank = 1
|
||||
|
||||
## create the snapshot itself
|
||||
##
|
||||
os.makedirs(snapshot.path)
|
||||
ignore_prefix = cfg.get("subvol", "ignore_prefix")
|
||||
|
||||
with chdir(cfg.get("subvol", "dir")):
|
||||
for sub in os.listdir():
|
||||
if sub.startswith(ignore_prefix):
|
||||
for subvolume in os.listdir():
|
||||
if subvolume.startswith(ignore_prefix):
|
||||
continue
|
||||
|
||||
os.system(shlex.join((
|
||||
"btrfs", "subvolume", "snapshot", "-r",
|
||||
sub,
|
||||
os.path.join(snap_path, sub)
|
||||
subvolume,
|
||||
os.path.join(snapshot.path, subvolume)
|
||||
)))
|
||||
|
||||
with open(os.path.join(snap_path, ".meta"), "w") as f:
|
||||
json.dump({
|
||||
"rank": rank
|
||||
}, f)
|
||||
snapshot.subvolumes.add(subvolume)
|
||||
|
||||
if rank == 2:
|
||||
print("Rank 2, serialize and GC now")
|
||||
## do optional processing
|
||||
##
|
||||
if snapshot.rank == 2:
|
||||
|
||||
def do_list(args, context):
|
||||
## snapshot serialization
|
||||
# if there's a previous Rank 2 snapshot, compute a diff against it
|
||||
# if not or if the process fails, demote this snap to Rank 1
|
||||
prev_r2_snaps = list_snapshots(2)
|
||||
|
||||
if prev_r2_snaps:
|
||||
prev_r2_snap = prev_r2_snaps[-1]
|
||||
print(">>> doing a diff against snapshot %s" %(prev_r2_snap.name))
|
||||
|
||||
error = serialize(
|
||||
snapshot,
|
||||
cfg.get("storage", "dir"),
|
||||
cfg.get("crypto", "keyfile"),
|
||||
prev_r2_snap
|
||||
)
|
||||
|
||||
if error:
|
||||
print("!!! serialization failed, demoting snapshot to Rank 1")
|
||||
snapshot.rank = 1
|
||||
|
||||
else:
|
||||
print("!!! no previous Rank 2 snapshot, please create one using btv stream")
|
||||
snapshot.rank = 1
|
||||
|
||||
## garbage collection
|
||||
do_gc()
|
||||
|
||||
## save all snapshot metadata
|
||||
snapshot.dump()
|
||||
|
||||
print(">>> Snapshot created: rank %d %s" %(snapshot.rank, snapshot.name))
|
||||
|
||||
def do_list(args):
|
||||
"""
|
||||
Print a list of existing snapshots.
|
||||
"""
|
||||
|
||||
print("R Name")
|
||||
print("Rank Name")
|
||||
|
||||
for snap in list_snapshots():
|
||||
print("%d %s" %(snap.rank, snap.name))
|
||||
print(" %d %s %s" %(snap.rank, snap.name, ", ".join(snap.notes)))
|
||||
|
||||
def do_drop(args, context):
|
||||
def do_drop(args):
|
||||
"""
|
||||
Drop a snapshot in args[0].
|
||||
"""
|
||||
|
||||
snap_name = args[0]
|
||||
snap = get_snapshot(snap_name)
|
||||
snap = get_snapshot_by_name(args[0])
|
||||
|
||||
if not snap:
|
||||
print("!!! %s is not a snapshot" %(snap_name))
|
||||
print("!!! %s is not a snapshot" %(args[0]))
|
||||
sys.exit(2)
|
||||
|
||||
snap_path = os.path.join(cfg.get("snap", "dir"), snap_name)
|
||||
error = snap.drop()
|
||||
|
||||
for dir in os.listdir(snap_path):
|
||||
if dir[0] != ".":
|
||||
btrfs_snap = os.path.join(snap_path, dir)
|
||||
os.system(shlex.join(("btrfs", "subvolume", "delete", btrfs_snap)))
|
||||
if error:
|
||||
print("!!! failed to drop snapshot %s" %(snap.name))
|
||||
|
||||
os.remove(os.path.join(snap_path, ".meta"))
|
||||
os.rmdir(snap_path)
|
||||
def do_stream(args):
|
||||
"""
|
||||
Stream the snapshot args[0] into dir args[1].
|
||||
|
||||
The affected snapshot is then promoted to Rank 2.
|
||||
"""
|
||||
|
||||
snapshot = get_snapshot_by_name(args[0])
|
||||
|
||||
if not snapshot:
|
||||
print("!!! %s is not a snapshot" %(args[0]))
|
||||
sys.exit(2)
|
||||
|
||||
directory = args[1]
|
||||
|
||||
if not os.path.isdir(directory):
|
||||
print("!!! %s is not a directory" %(directory))
|
||||
sys.exit(2)
|
||||
|
||||
print(">>> Serializing %s into %s" %(snapshot.name, directory))
|
||||
|
||||
error = serialize(
|
||||
snapshot,
|
||||
directory,
|
||||
cfg.get("crypto", "keyfile")
|
||||
)
|
||||
|
||||
if error:
|
||||
print("!!! serialization failed")
|
||||
else:
|
||||
snapshot.rank = 2
|
||||
snapshot.notes.add("fully streamed")
|
||||
snapshot.dump()
|
||||
|
||||
print("<<< %s serialized and promoted to Rank %d" %(snapshot.name, snapshot.rank))
|
||||
|
||||
def do_gc(args=None):
|
||||
"""
|
||||
Drops old snapshots.
|
||||
"""
|
||||
|
||||
counts = [0, 0, 0] # Rank 0, 1, 2
|
||||
limits = [
|
||||
cfg.getint("gc", "rank_0_count"),
|
||||
cfg.getint("gc", "rank_1_count"),
|
||||
cfg.getint("gc", "rank_2_count")
|
||||
]
|
||||
|
||||
for snap in reversed(list_snapshots()):
|
||||
counts[snap.rank] += 1
|
||||
|
||||
if counts[snap.rank] > limits[snap.rank]:
|
||||
print(" >> delete Rank %d %s" %(snap.rank, snap.name))
|
||||
|
||||
snap.drop()
|
||||
|
||||
verb_router = {
|
||||
"snapshot": (do_create, None),
|
||||
"list": (do_list, None),
|
||||
"drop": (do_drop, None),
|
||||
# ~ "prune": (do_prune, None),
|
||||
# ~ "stream-full": (do_stream, "full")
|
||||
"snapshot": do_create,
|
||||
"list": do_list,
|
||||
"drop": do_drop,
|
||||
"gc": do_gc,
|
||||
"stream": do_stream
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
|
|
@ -213,14 +430,20 @@ def print_help():
|
|||
{cmd} VERB [args]
|
||||
|
||||
Verbs:
|
||||
snapshot [--hooks]
|
||||
Create a snapshot. If --hooks is passed, do all optional work.
|
||||
snapshot [--process]
|
||||
Create a snapshot. If --process is passed, do all optional work.
|
||||
|
||||
stream SNAPSHOT OUTPUT_DIR
|
||||
Streams the SNAPSHOT into OUTPUT_DIR as a full (milestone) bin.
|
||||
|
||||
list
|
||||
List snapshots and their rank.
|
||||
|
||||
drop SNAPSHOT
|
||||
Drops (removes) SNAPSHOT. Use the full name as provided by {cmd} list.
|
||||
|
||||
gc
|
||||
Drops old local snapshots based on garbage collector settings.
|
||||
""".format(cmd=sys.argv[0]))
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
@ -235,10 +458,8 @@ if __name__ == "__main__":
|
|||
verify_user()
|
||||
verify_keyfile()
|
||||
|
||||
fn, ctx = verb_router[verb]
|
||||
|
||||
try:
|
||||
fn(sys.argv[2:], ctx)
|
||||
verb_router[verb](sys.argv[2:])
|
||||
except UsageError:
|
||||
print()
|
||||
print_help()
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
[main]
|
||||
[crypto]
|
||||
# Openssl password file (first line is the password).
|
||||
keyfile = /etc/btv/key
|
||||
|
||||
|
|
@ -22,9 +22,15 @@ rank_1_interval=6
|
|||
# comment holds.
|
||||
rank_2_interval=24
|
||||
|
||||
[prune]
|
||||
[storage]
|
||||
# Directory where serialized bins should be placed. This directory should
|
||||
# be one-way (write-only) synchronized to off-site storage, e.g. using
|
||||
# Syncthing or Grid.
|
||||
dir = /mnt/pool/subvol/_backup
|
||||
|
||||
[gc]
|
||||
# How many snapshots of each Rank to keep around.
|
||||
# Note: pruning is only performed when a new Rank 2 snapshot is created.
|
||||
# Note: pruning is only performed when a new Rank 2 snapshot is attempted.
|
||||
# Default values are 25, 36, and 42, i.e. 5 hours for Rank 0, 2 days for Rank 1,
|
||||
# and a week for Rank 2.
|
||||
rank_0_count=25
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue