flifloo/BackupPC_archive_host_B2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Update backup manager script

Browse source
This commit is contained in:
Ethanell 2022-10-18 00:02:58 +02:00
parent dca2dbdad8
commit 4cbaf0c52f
1 changed files with 300 additions and 333 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

633
backup-manager.py Executable file → Normal file
View file

@ -1,6 +1,6 @@
#!/usr/bin/python -W ignore::DeprecationWarning #!/usr/bin/python3
# #
# Script to manage S3-stored backups # Script to manage B2-stored backups
# #
# Copyright (c) 2009-2013 Ryan S. Tucker # Copyright (c) 2009-2013 Ryan S. Tucker
# #
@ -22,33 +22,37 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE. # THE SOFTWARE.
import optparse from argparse import ArgumentParser
import os
import pwd
import secrets
import sys
import time
from boto.s3.connection import S3Connection
from collections import defaultdict from collections import defaultdict
from math import log10 from datetime import datetime, timedelta
from os import environ, getuid
from pathlib import Path
from pwd import getpwuid
from subprocess import Popen from subprocess import Popen
from sys import argv
from b2sdk.v2 import FileVersion
from b2sdk.v2 import InMemoryAccountInfo, B2Api, Bucket
from math import log10
from progress.bar import ChargingBar
import secrets
class BackupManager: class BackupManager:
def __init__(self, access_key: str, shared_key: str):
def __init__(self, accesskey, sharedkey): self._access_key = access_key
self._accesskey = accesskey b2_info = InMemoryAccountInfo()
self._connection = S3Connection(accesskey, sharedkey) self._connection = B2Api(b2_info)
self._connection.authorize_account("production", self._access_key, shared_key)
self._buckets = None self._buckets = None
self._bucketbackups = {} self._bucket_backups = {}
self._backups = None self._backups = None
def _generate_backup_buckets(self): def _generate_backup_buckets(self) -> [Bucket]:
bucket_prefix = self._accesskey.lower() + '-bkup-' bucket_prefix = f"{self._access_key}-bckpc-".lower()
buckets = self._connection.get_all_buckets() buckets = self._connection.list_buckets()
self._buckets = [] self._buckets = []
for bucket in buckets: for bucket in buckets:
@ -56,20 +60,24 @@ class BackupManager:
self._buckets.append(bucket) self._buckets.append(bucket)
@property @property
def backup_buckets(self): # property def backup_buckets(self) -> [Bucket]:
if self._buckets is None: if self._buckets is None:
self._generate_backup_buckets() self._generate_backup_buckets()
return self._buckets return self._buckets
def _list_backups(self, bucket): @staticmethod
"""Returns a dict of backups in a bucket, with dicts of: def _list_backups(bucket: Bucket) -> {}:
"""
Returns a dict of backups in a bucket, with dicts of:
{hostname (str): {hostname (str):
{Backup number (int): {Backup number (int):
{'date': Timestamp of backup (int), {
'keys': A list of keys comprising the backup, "date": Datetime of backup (int),
'hostname': Hostname (str), "files": A list of files comprising the backup,
'backupnum': Backup number (int), "hostname": Hostname (str),
'finalized': 0, or the timestamp the backup was finalized "backup_num": Backup number (int),
"finalized": 0, or the timestamp the backup was finalized,
"bucket": the bucket of the backup
} }
} }
} }
@ -77,120 +85,123 @@ class BackupManager:
backups = {} backups = {}
for key in bucket.list(): for file in filter(lambda e: isinstance(e, FileVersion), map(lambda e: e[0], bucket.ls())):
keyparts = key.key.split('.') file: FileVersion = file
parts = file.file_name.split(".")
final = False final = False
if keyparts[-1] == 'COMPLETE': if parts[-1] == "COMPLETE":
final = True final = True
keyparts.pop() # back to tar parts.pop() # back to tar
keyparts.pop() # back to backup number parts.pop() # back to backup number
else: else:
if keyparts[-1] == 'gpg': if parts[-1] == "gpg":
keyparts.pop() parts.pop()
if keyparts[-1] != 'tar' and len(keyparts[-1]) is 2: if parts[-1] != "tar" and len(parts[-1]) == 2:
keyparts.pop() parts.pop()
if keyparts[-1] == 'tar': if parts[-1] == "tar":
keyparts.pop() parts.pop()
nextpart = keyparts.pop() nextpart = parts.pop()
if nextpart == 'COMPLETE': if nextpart == "COMPLETE":
print("Stray file: %s" % key.key) print(f"Stray file: {file.file_name}")
continue continue
backupnum = int(nextpart) backup_num = int(nextpart)
hostname = '.'.join(keyparts) hostname = ".".join(parts)
lastmod = time.strptime(key.last_modified, upload_timestamp = file.upload_timestamp//1000
'%Y-%m-%dT%H:%M:%S.000Z') lastmod = datetime.utcfromtimestamp(upload_timestamp)
if hostname in backups.keys(): if hostname in backups.keys():
if not backupnum in backups[hostname].keys(): if backup_num not in backups[hostname].keys():
backups[hostname][backupnum] = { backups[hostname][backup_num] = {
'date': lastmod, "date": lastmod,
'hostname': hostname, "hostname": hostname,
'backupnum': backupnum, "backup_num": backup_num,
'finalized': 0, "finalized": 0,
'keys': [], "files": [],
'finalkey': None, "final_file": None,
'finalized_age': -1, "finalized_age": -1,
"bucket": bucket
} }
else: else:
backups[hostname] = { backups[hostname] = {
backupnum: { backup_num: {
'date': lastmod, "date": lastmod,
'hostname': hostname, "hostname": hostname,
'backupnum': backupnum, "backup_num": backup_num,
'finalized': 0, "finalized": 0,
'keys': [], "files": [],
'finalkey': None, "final_file": None,
'finalized_age': -1, "finalized_age": -1,
"bucket": bucket
} }
} }
if final: if final:
backups[hostname][backupnum]['finalized'] = lastmod backups[hostname][backup_num]["finalized"] = upload_timestamp
backups[hostname][backupnum]['finalkey'] = key backups[hostname][backup_num]["final_file"] = file
timestamp = time.mktime(lastmod)
delta = int(time.time() - timestamp + time.timezone) delta = int((lastmod - datetime.now()).total_seconds() * 1000000)
backups[hostname][backupnum]['finalized_age'] = delta backups[hostname][backup_num]["finalized_age"] = delta
else: else:
if lastmod < backups[hostname][backupnum]['date']: if lastmod < backups[hostname][backup_num]["date"]:
backups[hostname][backupnum]['date'] = lastmod backups[hostname][backup_num]["date"] = lastmod
backups[hostname][backupnum]['keys'].append(key) backups[hostname][backup_num]["files"].append(file)
return backups return backups
def get_backups_by_bucket(self, bucket): def get_backups_by_bucket(self, bucket: Bucket) -> {}:
if bucket.name not in self._bucketbackups: if bucket.name not in self._bucket_backups:
self._bucketbackups[bucket.name] = self._list_backups(bucket) self._bucket_backups[bucket.name] = self._list_backups(bucket)
return self._bucketbackups[bucket.name] return self._bucket_backups[bucket.name]
@property @property
def all_backups(self): # property def all_backups(self) -> [{}]:
if self._backups is None: if self._backups is None:
sys.stderr.write("Enumerating backups")
self._backups = {} self._backups = {}
for bucket in self.backup_buckets: for bucket in self.backup_buckets:
backups_dict = self.get_backups_by_bucket(bucket) backups_dict = self.get_backups_by_bucket(bucket)
for hostname, backups in backups_dict.items(): for hostname, backups in backups_dict.items():
sys.stderr.write('.')
sys.stderr.flush()
if hostname not in self._backups: if hostname not in self._backups:
self._backups[hostname] = {} self._backups[hostname] = {}
self._backups[hostname].update(backups) self._backups[hostname].update(backups)
sys.stderr.write("\n")
return self._backups return self._backups
def invalidate_host_cache(self, hostname): def invalidate_host_cache(self, hostname):
nuke = [] nuke = []
for bucket in self._bucketbackups: for bucket in self._bucket_backups:
if hostname in self._bucketbackups[bucket]: if hostname in self._bucket_backups[bucket]:
nuke.append(bucket) nuke.append(bucket)
for bucket in nuke: for bucket in nuke:
if bucket in self._bucketbackups: if bucket in self._bucket_backups:
del self._bucketbackups[bucket] del self._bucket_backups[bucket]
self._backups = None self._backups = None
@property @property
def backups_by_age(self): # property def backups_by_age(self):
"Returns a dict of {hostname: [(backupnum, age), ...]}" """
Returns a dict of {hostname: [(backup_num, age), ...]}
"""
results = defaultdict(list) results = defaultdict(list)
for hostname, backups in self.all_backups.items(): for hostname, backups in self.all_backups.items():
for backupnum, statusdict in backups.items(): for backup_num, statusdict in backups.items():
results[hostname].append((backupnum, results[hostname].append((backup_num,
statusdict['finalized_age'])) statusdict["finalized_age"]))
return results return results
def choose_host_to_backup(agedict, target_count=2): def choose_host_to_backup(age_dict, target_count=2):
"Takes a dict from backups_by_age, returns a hostname to back up." """
Takes a dict from backups_by_age, returns a hostname to back up.
"""
host_scores = defaultdict(int) host_scores = defaultdict(int)
for hostname, backuplist in agedict.items(): for hostname, backup_list in age_dict.items():
bl = sorted(backuplist, key=lambda x: x[1]) bl = sorted(backup_list, key=lambda x: x[1])
if len(bl) > 0 and bl[0][1] == -1: if len(bl) > 0 and bl[0][1] == -1:
# unfinalized backup alert # unfinalized backup alert
host_scores[hostname] += 200 host_scores[hostname] += 200
@ -199,20 +210,22 @@ def choose_host_to_backup(agedict, target_count=2):
host_scores[hostname] -= 100 host_scores[hostname] -= 100
host_scores[hostname] -= len(bl) host_scores[hostname] -= len(bl)
if len(bl) > 0: if len(bl) > 0:
# age of oldest backup helps score # age of the oldest backup helps score
oldest = bl[0] oldest = bl[0]
host_scores[hostname] += log10(oldest[1]) host_scores[hostname] += log10(oldest[1])
# recency of newest backup hurts score # recency of the newest backup hurts score
newest = bl[-1] newest = bl[-1]
host_scores[hostname] -= log10(max(1, (oldest[1] - newest[1]))) host_scores[hostname] -= log10(max(1, (oldest[1] - newest[1])))
for candidate, score in sorted(host_scores.items(), for candidate, score in sorted(host_scores.items(),
key=lambda x: x[1], reverse=True): key=lambda x: x[1], reverse=True):
yield (candidate, score) yield candidate, score
def choose_backups_to_delete(agedict, target_count=2, max_age=30): def choose_backups_to_delete(agedict, target_count=2, max_age=30):
"Takes a dict from backups_by_age, returns a list of backups to delete" """
Takes a dict from backups_by_age, returns a list of backups to delete
"""
decimate = defaultdict(list) decimate = defaultdict(list)
@ -230,279 +243,233 @@ def choose_backups_to_delete(agedict, target_count=2, max_age=30):
return decimate return decimate
def iter_urls(keyset, expire=86400): def make_restore_script(backup: {}, expire=86400) -> str:
"""Given a list of keys and an optional expiration time (in seconds), """
returns an iterator of URLs to fetch to reassemble the backup.""" Returns a quick and easy restoration script to restore the given system,
requires a backup, and perhaps expire
"""
for key in keyset: hostname = backup["hostname"]
yield key.generate_url(expires_in=expire) backup_num = backup["backup_num"]
bucket = backup["bucket"]
friendly_time = backup["date"].strftime("%Y-%m-%d at %H:%M GMT")
expire_time = datetime.now() + timedelta(seconds=expire)
def make_restore_script(backup, expire=86400): files = [f"'{bucket.get_download_url(i.file_name)}'" for i in backup["files"] + [backup["final_file"]]]
"""Returns a quick and easy restoration script to restore the given system,
requires a backup, and perhaps expire"""
myhostname = backup['hostname'] output = f"""#!/bin/bash
mybackupnum = backup['backupnum'] # Restoration script for {hostname} backup {backup_num},
myfriendlytime = time.strftime('%Y-%m-%d at %H:%M GMT', backup['date']) # a backup created on {friendly_time}.
myexpiretime = time.strftime('%Y-%m-%d at %H:%M GMT', # To use: bash scriptname /path/to/put/the/files
time.gmtime(time.time() + expire))
myexpiretimestamp = time.time() + expire
output = [] # WARNING: THIS FILE EXPIRES AFTER {expire_time.strftime("%Y-%m-%d at %H:%M GMT")}
if (( "$(date +%s)" > "{int(expire_time.timestamp() * 1000000)}" )); then
echo "Sorry, this restore script is too old."
exit 1
elif [ -z "$1" ]; then
echo "Usage: ./scriptname /path/to/restore/to"
exit 1
elif [ ! -d "$1" ]; then
echo "Target $1 does not exist!"
exit 1
elif [ -n "$(ls --almost-all "$1")" ]; then
echo "Target $1 is not empty!"
exit 1
fi
output.append('#!/bin/sh\n') # cd to the destination, create a temporary workspace
output.append('# Restoration script for %s backup %s,\n' % ( cd "$1"
myhostname, mybackupnum)) tmp_dir="$i/.restorescript-scratch"
output.append('# a backup created on %s.\n' % (myfriendlytime)) mkdir "$tmp_dir"
output.append('# To use: bash scriptname /path/to/put/the/files\n\n')
output.append('# WARNING: THIS FILE EXPIRES AFTER %s\n' % (myexpiretime))
output.append('if [ "`date +%%s`" -gt "%i" ];\n' % (myexpiretimestamp))
output.append(' then echo "Sorry, this restore script is too old.";\n')
output.append(' exit 1;\n')
output.append('fi\n\n')
output.append('if [ -z "$1" ];\n')
output.append(' then echo "Usage: ./scriptname /path/to/restore/to";\n')
output.append(' exit 1;\n')
output.append('fi\n\n')
output.append('# Check the destination\n')
output.append('if [ ! -d $1 ];\n')
output.append(' then echo "Target $1 does not exist!";\n')
output.append(' exit 1;\n')
output.append('fi\n\n')
output.append('if [ -n "`ls --almost-all $1`" ];\n')
output.append(' then echo "Target $1 is not empty!";\n')
output.append(' exit 1;\n')
output.append('fi\n\n')
output.append('# cd to the destination, create a temporary workspace\n')
output.append('cd $1\n')
output.append('mkdir .restorescript-scratch\n\n')
output.append('# retrieve files\n')
mysortedfilelist = [] files=({' '.join(files)})
for key in backup['keys']: token='{bucket.get_download_authorization(f'{hostname}.{backup_num}', expire)}'
output.append('wget -O $1/.restorescript-scratch/%s "%s"\n' % (
key.name, key.generate_url(expires_in=expire)))
mysortedfilelist.append('.restorescript-scratch/' + key.name)
mysortedfilelist.sort()
output.append('\n# decrypt files\n') declare –a out_files
output.append('gpg --decrypt-files << EOF\n') for i in "${{files[@]}}"; do
output.append('\n'.join(mysortedfilelist)) filename="$(echo "$i" | cut -d/ -f6)"
output.append('\nEOF\n') curl "$i" -o "$tmp_dir/$filename" -H "Authorization: $token"
if (( $? != 0 )); then
echo "Error during download !"
exit 1
fi
out_files+=("$tmp_dir/$filename")
done
output.append('\n# join and untar files\n') # decrypt files
output.append('cat .restorescript-scratch/*.tar.?? | tar -xf -\n\n') gpg --decrypt-files "${{out_files[@]}}"
output.append('echo "DONE! Have a nice day."\n##\n') # join and untar files
cat "$tmp_dir/*.tar.??" | tar -xf -
echo "DONE! Have a nice day."
"""
return output return output
def start_archive(hosts): def start_archive(hosts):
"Starts an archive operation for a list of hosts." """
if 'LOGNAME' in os.environ: Starts an archive operation for a list of hosts.
username = os.environ['LOGNAME'] """
if "LOGNAME" in environ:
username = environ["LOGNAME"]
else: else:
try: try:
username = pwd.getpwuid(os.getuid()).pw_name username = getpwuid(getuid()).pw_name
except KeyError: except KeyError:
username = 'nobody' username = "nobody"
scriptdir = os.path.dirname(sys.argv[0]) cmd = [Path(argv[0]).parents[0] / "BackupPC_archiveStart", "archives3", username]
cmd = [os.path.join(scriptdir, 'BackupPC_archiveStart'), 'archives3',
username]
cmd.extend(hosts) cmd.extend(hosts)
proc = Popen(cmd) proc = Popen(cmd)
proc.communicate() proc.communicate()
def main(): def script(parser: ArgumentParser, bmgr: BackupManager, host: str, unfinalized: bool, backup_num: int = None,
# check command line options expire: int = 86400, filename: str = None):
parser = optparse.OptionParser( if not backup_num and unfinalized:
usage="usage: %prog [options] [list|delete|script]", # assuming highest number
description="" + backup_num = max(bmgr.all_backups[host].keys())
"Companion maintenance script for BackupPC_archiveHost_s3. " + elif not backup_num:
"By default, it assumes the 'list' command, which displays all " + # assuming highest finalized number
"of the backups currently archived on S3. The 'delete' command " + backup_num = 0
"is used to delete backups. The 'script' command produces a " + for backup in bmgr.all_backups[host].keys():
"script that can be used to download and restore a backup.") if bmgr.all_backups[host][backup]["finalized"] > 0:
parser.add_option("-H", "--host", dest="host", backup_num = max(backup_num, backup)
help="Name of backed-up host") if backup_num == 0:
parser.add_option("-b", "--backup-number", dest="backupnum", parser.error("No finalized backups found! Try --unfinalized if you dare")
help="Backup number")
parser.add_option("-a", "--age", dest="age",
help="Delete backups older than AGE days")
parser.add_option("-k", "--keep", dest="keep",
help="When used with --age, keep this many recent " +
"backups (default=1)", default=1)
parser.add_option("-f", "--filename", dest="filename",
help="Output filename for script")
parser.add_option("-x", "--expire", dest="expire",
help="Maximum age of script, default 86400 seconds")
parser.add_option("-t", "--test", dest="test", action="store_true",
help="Test mode; don't actually delete")
parser.add_option("-u", "--unfinalized", dest="unfinalized",
action="store_true", help="Consider unfinalized backups")
parser.add_option("-s", "--start-backups", dest="start",
action="store_true",
help="When used with --age, start backups for hosts " +
"with fewer than keep+1 backups")
parser.add_option("-l", "--list", dest="list", action="store_true",
help="List stored backups after completing operations")
(options, args) = parser.parse_args() backup = bmgr.all_backups[host][backup_num]
bmgr = BackupManager(secrets.accesskey, secrets.sharedkey) if filename:
with open(filename, "w") as fd:
if options.backupnum and not options.host: fd.write(make_restore_script(backup, expire=expire))
parser.error('Must specify --host when specifying --backup-number')
if options.backupnum:
options.backupnum = int(options.backupnum)
if len(args) == 0:
args.append('list')
if len(args) > 1:
parser.error('Too many arguments.')
if args[0] != 'delete' and options.age:
parser.error('--age only makes sense with delete')
if options.start and not (args[0] == 'delete' and options.age):
parser.error('--start-backups only makes sense with delete and --age')
if args[0] != 'script' and (options.expire or options.filename):
parser.error('--expire and --filename only make sense with script')
if args[0] in ['list', 'script', 'delete']:
if options.host:
if options.host not in bmgr.all_backups:
parser.error('No backups found for host "%s"' % options.host)
else:
if len(bmgr.all_backups) == 0:
parser.error('No buckets found!')
else: else:
parser.error('Invalid option: %s' + args[0]) print(make_restore_script(backup, expire=expire))
if args[0] == 'script':
if not options.host:
parser.error('Must specify --host to generate a script for')
if not options.backupnum and options.unfinalized: def delete(bm: BackupManager, keep: int, host: str, backup_num: int, age: int, test: bool,
# assuming highest number start: bool):
options.backupnum = max(bmgr.all_backups[options.host].keys()) to_delete = []
elif not options.backupnum: if host and backup_num:
# assuming highest finalized number print(f"Will delete backup: {host} {backup_num} (forced)")
options.backupnum = 0 to_delete.append((host, backup_num))
for backup in bmgr.all_backups[options.host].keys(): elif age:
if bmgr.all_backups[options.host][backup]['finalized'] > 0: to_delete_dict = choose_backups_to_delete(bm.backups_by_age, target_count=keep, max_age=age)
options.backupnum = max(options.backupnum, backup) for hostname, backup_list in to_delete_dict.items():
if options.backupnum == 0: for backup_stat in backup_list:
parser.error('No finalized backups found! Try ' print(f"Will delete backup: {hostname} {backup_stat[0]} (expired at {backup_stat[1] / 86400.0} days)")
'--unfinalized if you dare') to_delete.append((hostname, backup_stat[0]))
else:
return
backup = bmgr.all_backups[options.host][options.backupnum] for delete_host, delete_backup_num in to_delete:
host_backups = bm.all_backups.get(delete_host, {})
delete_backup = host_backups.get(delete_backup_num, {})
delete_files = delete_backup.get("files", [])
final_file = delete_backup.get("final_file", None)
if len(delete_files) > 0:
for file in ChargingBar(f"Deleting backup {delete_host} #{delete_backup_num}:", max=len(delete_files)).\
iter(delete_files):
if not test:
file.delete()
if not options.expire: if final_file and not test:
options.expire = "86400" final_file.delete()
if options.filename: if start:
fd = open(options.filename, 'w') for delete_host, delete_backup_num in to_delete:
fd.writelines(make_restore_script(backup, bm.invalidate_host_cache(delete_host)
expire=int(options.expire))) score_iter = choose_host_to_backup(bm.backups_by_age, target_count=int(keep) + 1)
for candidate, score in score_iter:
if score > 0:
print(f"Starting archive operation for host: {candidate} (score={score})")
start_archive([candidate])
break
def list_backups(bm: BackupManager):
print(f"{'Hostname':>25} | {'Bkup#':>5} | {'Age':>30} | {'Files':>5}")
print(("-" * 72))
for hostname, backups in bm.all_backups.items():
for backup_num in sorted(backups.keys()):
filecount = len(backups[backup_num]["files"])
date = backups[backup_num]["date"]
if backups[backup_num]["finalized"] > 0:
in_progress = ""
else:
in_progress = "*"
print(f"{hostname:>25} | {backup_num:>5} | {str(datetime.now() - date):>30} | {filecount:>5}{in_progress}")
print("* = not yet finalized (Age = time of last activity)")
def main():
parser = ArgumentParser(description="Companion maintenance script for BackupPC_archiveHost_s3. " +
"By default, it assumes the 'list' command, which displays all " +
"of the backups currently archived on B2. The 'delete' command " +
"is used to delete backups. The 'script' command produces a " +
"script that can be used to download and restore a backup.")
parser.add_argument("-l", "--list", dest="list", action="store_true",
help="List stored backups after completing operations")
subparsers = parser.add_subparsers(required=True, dest="action")
subparsers.add_parser("list")
delete_parser = subparsers.add_parser("delete")
delete_parser.add_argument("-s", "--start-backups", dest="start", action="store_true",
help="When used with --age, start backups for hosts with fewer than keep+1 backups")
delete_parser.add_argument("-k", "--keep", dest="keep", help="When used with --age, keep this many recent backups",
default=1)
delete_parser.add_argument("-t", "--test", dest="test", action="store_true",
help="Test mode; don't actually delete")
delete_parser.add_argument("-H", "--host", dest="host", help="Name of backed-up host")
delete_parser.add_argument("-b", "--backup-number", dest="backup_num", type=int, help="Backup number")
delete_parser.add_argument("-a", "--age", dest="age", help="Delete backups older than AGE days")
script_parser = subparsers.add_parser("script")
script_parser.add_argument("-H", "--host", dest="host", required=True, help="Name of backed-up host")
script_parser.add_argument("-b", "--backup-number", dest="backup_num", type=int, help="Backup number")
script_parser.add_argument("-f", "--filename", dest="filename", help="Output filename for script")
script_parser.add_argument("-x", "--expire", dest="expire", default=86400, help="Maximum age of script")
script_parser.add_argument("-u", "--unfinalized", dest="unfinalized", action="store_true",
help="Consider unfinalized backups")
args = parser.parse_args()
bm = BackupManager(secrets.access_key, secrets.shared_key)
if args.action == "script" or args.action == "delete":
if args.backup_num and not args.host:
parser.error("Must specify --host when specifying --backup-number")
if args.host:
if args.host not in bm.all_backups:
parser.error(f"No backups found for host \"{args.host}\"")
else: else:
sys.stdout.writelines(make_restore_script(backup, if len(bm.all_backups) == 0:
expire=int(options.expire))) parser.error("No buckets found!")
elif args[0] == 'delete':
to_ignore = int(options.keep)
to_delete = []
if options.host and options.backupnum:
print("Will delete backup: %s %i (forced)" % (
options.host, options.backupnum))
to_delete.append((options.host, options.backupnum))
elif options.age:
to_delete_dict = choose_backups_to_delete(bmgr.backups_by_age,
target_count=to_ignore,
max_age=int(options.age))
for hostname, backuplist in to_delete_dict.items():
for backupstat in backuplist:
print("Will delete backup: %s %i (expired at %g days)" % (
hostname, backupstat[0], backupstat[1] / 86400.0))
to_delete.append((hostname, backupstat[0]))
else: if args.action == "script":
parser.error('Need either an age or a host AND backup number.') script(parser, bm, args.host, args.backup_num, args.unfinalized, args.expire, args.filename)
elif args.action == "delete":
if not (args.age or args.host or args.backup_num):
parser.error("--age or --host and --backup-number are required")
elif args.host and not args.backup_num:
parser.error("--backup-number required with --host")
elif args.age and (args.host or args.backup_num):
parser.error("--age can't be combined with --host or --backup-number")
elif args.start and not args.age:
parser.error("--start-backups only makes sense with --age")
if len(to_delete) > 0: delete(bm, args.keep, args.host, args.backup_num, args.age, args.test, args.start)
for deletehost, deletebackupnum in to_delete:
hostbackups = bmgr.all_backups.get(deletehost, {})
deletebackup = hostbackups.get(deletebackupnum, {})
deletekeys = deletebackup.get('keys', [])
finalkey = deletebackup.get('finalkey', None)
if len(deletekeys) > 0:
sys.stdout.write("Deleting backup: %s %d (%d keys)" % (
deletehost, deletebackupnum, len(deletekeys)))
for key in deletekeys:
if options.test:
sys.stdout.write('_')
else:
key.delete()
sys.stdout.write('.')
sys.stdout.flush()
if finalkey is not None:
if options.test:
sys.stdout.write('+')
else:
finalkey.delete()
sys.stdout.write('!')
sys.stdout.flush()
sys.stdout.write('\n')
if options.start: if args.action == "list" or args.list:
for deletehost, deletebackupnum in to_delete: list_backups(bm)
bmgr.invalidate_host_cache(deletehost)
score_iter = choose_host_to_backup(bmgr.backups_by_age,
target_count=int(options.keep) + 1)
for candidate, score in score_iter:
if score > 0:
sys.stdout.write('Starting archive operation for host: '
'%s (score=%g)\n' % (candidate, score))
start_archive([candidate])
break
if args[0] == 'list' or options.list:
sys.stdout.write('%25s | %5s | %20s | %5s\n' % (
"Hostname", "Bkup#", "Age", "Files"))
sys.stdout.write(('-' * 72) + '\n')
for hostname, backups in bmgr.all_backups.items():
for backupnum in sorted(backups.keys()):
filecount = len(backups[backupnum]['keys'])
datestruct = backups[backupnum]['date']
if backups[backupnum]['finalized'] > 0:
inprogress = ''
else:
inprogress = '*'
timestamp = time.mktime(datestruct)
delta = int(time.time() - timestamp + time.timezone)
if delta < 3600:
prettydelta = '%i min ago' % (delta / 60)
elif delta < 86400:
prettydelta = '%i hr ago' % (delta / 3600)
else:
days = int(delta / 60 / 60 / 24)
if days == 1:
s = ''
else:
s = 's'
prettydelta = '%i day%s ago' % (days, s)
sys.stdout.write('%25s | %5i | %20s | %5i%s\n' % (
hostname, backupnum, prettydelta, filecount, inprogress))
sys.stdout.write('* == not yet finalized (Age == time of '
'last activity)\n')
if __name__ == '__main__': if __name__ == "__main__":
main() main()