Update backup manager script
This commit is contained in:
parent
dca2dbdad8
commit
4cbaf0c52f
1 changed files with 300 additions and 333 deletions
631
backup-manager.py
Executable file → Normal file
631
backup-manager.py
Executable file → Normal file
|
|
@ -1,6 +1,6 @@
|
|||
#!/usr/bin/python -W ignore::DeprecationWarning
|
||||
#!/usr/bin/python3
|
||||
#
|
||||
# Script to manage S3-stored backups
|
||||
# Script to manage B2-stored backups
|
||||
#
|
||||
# Copyright (c) 2009-2013 Ryan S. Tucker
|
||||
#
|
||||
|
|
@ -22,33 +22,37 @@
|
|||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import optparse
|
||||
import os
|
||||
import pwd
|
||||
import secrets
|
||||
import sys
|
||||
import time
|
||||
|
||||
from boto.s3.connection import S3Connection
|
||||
|
||||
from argparse import ArgumentParser
|
||||
from collections import defaultdict
|
||||
from math import log10
|
||||
from datetime import datetime, timedelta
|
||||
from os import environ, getuid
|
||||
from pathlib import Path
|
||||
from pwd import getpwuid
|
||||
from subprocess import Popen
|
||||
from sys import argv
|
||||
|
||||
from b2sdk.v2 import FileVersion
|
||||
from b2sdk.v2 import InMemoryAccountInfo, B2Api, Bucket
|
||||
from math import log10
|
||||
from progress.bar import ChargingBar
|
||||
|
||||
import secrets
|
||||
|
||||
|
||||
class BackupManager:
|
||||
|
||||
def __init__(self, accesskey, sharedkey):
|
||||
self._accesskey = accesskey
|
||||
self._connection = S3Connection(accesskey, sharedkey)
|
||||
def __init__(self, access_key: str, shared_key: str):
|
||||
self._access_key = access_key
|
||||
b2_info = InMemoryAccountInfo()
|
||||
self._connection = B2Api(b2_info)
|
||||
self._connection.authorize_account("production", self._access_key, shared_key)
|
||||
|
||||
self._buckets = None
|
||||
self._bucketbackups = {}
|
||||
self._bucket_backups = {}
|
||||
self._backups = None
|
||||
|
||||
def _generate_backup_buckets(self):
|
||||
bucket_prefix = self._accesskey.lower() + '-bkup-'
|
||||
buckets = self._connection.get_all_buckets()
|
||||
def _generate_backup_buckets(self) -> [Bucket]:
|
||||
bucket_prefix = f"{self._access_key}-bckpc-".lower()
|
||||
buckets = self._connection.list_buckets()
|
||||
self._buckets = []
|
||||
|
||||
for bucket in buckets:
|
||||
|
|
@ -56,20 +60,24 @@ class BackupManager:
|
|||
self._buckets.append(bucket)
|
||||
|
||||
@property
|
||||
def backup_buckets(self): # property
|
||||
def backup_buckets(self) -> [Bucket]:
|
||||
if self._buckets is None:
|
||||
self._generate_backup_buckets()
|
||||
return self._buckets
|
||||
|
||||
def _list_backups(self, bucket):
|
||||
"""Returns a dict of backups in a bucket, with dicts of:
|
||||
@staticmethod
|
||||
def _list_backups(bucket: Bucket) -> {}:
|
||||
"""
|
||||
Returns a dict of backups in a bucket, with dicts of:
|
||||
{hostname (str):
|
||||
{Backup number (int):
|
||||
{'date': Timestamp of backup (int),
|
||||
'keys': A list of keys comprising the backup,
|
||||
'hostname': Hostname (str),
|
||||
'backupnum': Backup number (int),
|
||||
'finalized': 0, or the timestamp the backup was finalized
|
||||
{
|
||||
"date": Datetime of backup (int),
|
||||
"files": A list of files comprising the backup,
|
||||
"hostname": Hostname (str),
|
||||
"backup_num": Backup number (int),
|
||||
"finalized": 0, or the timestamp the backup was finalized,
|
||||
"bucket": the bucket of the backup
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -77,120 +85,123 @@ class BackupManager:
|
|||
|
||||
backups = {}
|
||||
|
||||
for key in bucket.list():
|
||||
keyparts = key.key.split('.')
|
||||
for file in filter(lambda e: isinstance(e, FileVersion), map(lambda e: e[0], bucket.ls())):
|
||||
file: FileVersion = file
|
||||
parts = file.file_name.split(".")
|
||||
final = False
|
||||
|
||||
if keyparts[-1] == 'COMPLETE':
|
||||
if parts[-1] == "COMPLETE":
|
||||
final = True
|
||||
keyparts.pop() # back to tar
|
||||
keyparts.pop() # back to backup number
|
||||
parts.pop() # back to tar
|
||||
parts.pop() # back to backup number
|
||||
else:
|
||||
if keyparts[-1] == 'gpg':
|
||||
keyparts.pop()
|
||||
if parts[-1] == "gpg":
|
||||
parts.pop()
|
||||
|
||||
if keyparts[-1] != 'tar' and len(keyparts[-1]) is 2:
|
||||
keyparts.pop()
|
||||
if parts[-1] != "tar" and len(parts[-1]) == 2:
|
||||
parts.pop()
|
||||
|
||||
if keyparts[-1] == 'tar':
|
||||
keyparts.pop()
|
||||
if parts[-1] == "tar":
|
||||
parts.pop()
|
||||
|
||||
nextpart = keyparts.pop()
|
||||
if nextpart == 'COMPLETE':
|
||||
print("Stray file: %s" % key.key)
|
||||
nextpart = parts.pop()
|
||||
if nextpart == "COMPLETE":
|
||||
print(f"Stray file: {file.file_name}")
|
||||
continue
|
||||
backupnum = int(nextpart)
|
||||
hostname = '.'.join(keyparts)
|
||||
backup_num = int(nextpart)
|
||||
hostname = ".".join(parts)
|
||||
|
||||
lastmod = time.strptime(key.last_modified,
|
||||
'%Y-%m-%dT%H:%M:%S.000Z')
|
||||
upload_timestamp = file.upload_timestamp//1000
|
||||
lastmod = datetime.utcfromtimestamp(upload_timestamp)
|
||||
|
||||
if hostname in backups.keys():
|
||||
if not backupnum in backups[hostname].keys():
|
||||
backups[hostname][backupnum] = {
|
||||
'date': lastmod,
|
||||
'hostname': hostname,
|
||||
'backupnum': backupnum,
|
||||
'finalized': 0,
|
||||
'keys': [],
|
||||
'finalkey': None,
|
||||
'finalized_age': -1,
|
||||
if backup_num not in backups[hostname].keys():
|
||||
backups[hostname][backup_num] = {
|
||||
"date": lastmod,
|
||||
"hostname": hostname,
|
||||
"backup_num": backup_num,
|
||||
"finalized": 0,
|
||||
"files": [],
|
||||
"final_file": None,
|
||||
"finalized_age": -1,
|
||||
"bucket": bucket
|
||||
}
|
||||
else:
|
||||
backups[hostname] = {
|
||||
backupnum: {
|
||||
'date': lastmod,
|
||||
'hostname': hostname,
|
||||
'backupnum': backupnum,
|
||||
'finalized': 0,
|
||||
'keys': [],
|
||||
'finalkey': None,
|
||||
'finalized_age': -1,
|
||||
backup_num: {
|
||||
"date": lastmod,
|
||||
"hostname": hostname,
|
||||
"backup_num": backup_num,
|
||||
"finalized": 0,
|
||||
"files": [],
|
||||
"final_file": None,
|
||||
"finalized_age": -1,
|
||||
"bucket": bucket
|
||||
}
|
||||
}
|
||||
if final:
|
||||
backups[hostname][backupnum]['finalized'] = lastmod
|
||||
backups[hostname][backupnum]['finalkey'] = key
|
||||
timestamp = time.mktime(lastmod)
|
||||
delta = int(time.time() - timestamp + time.timezone)
|
||||
backups[hostname][backupnum]['finalized_age'] = delta
|
||||
backups[hostname][backup_num]["finalized"] = upload_timestamp
|
||||
backups[hostname][backup_num]["final_file"] = file
|
||||
|
||||
delta = int((lastmod - datetime.now()).total_seconds() * 1000000)
|
||||
backups[hostname][backup_num]["finalized_age"] = delta
|
||||
else:
|
||||
if lastmod < backups[hostname][backupnum]['date']:
|
||||
backups[hostname][backupnum]['date'] = lastmod
|
||||
backups[hostname][backupnum]['keys'].append(key)
|
||||
if lastmod < backups[hostname][backup_num]["date"]:
|
||||
backups[hostname][backup_num]["date"] = lastmod
|
||||
backups[hostname][backup_num]["files"].append(file)
|
||||
return backups
|
||||
|
||||
def get_backups_by_bucket(self, bucket):
|
||||
if bucket.name not in self._bucketbackups:
|
||||
self._bucketbackups[bucket.name] = self._list_backups(bucket)
|
||||
def get_backups_by_bucket(self, bucket: Bucket) -> {}:
|
||||
if bucket.name not in self._bucket_backups:
|
||||
self._bucket_backups[bucket.name] = self._list_backups(bucket)
|
||||
|
||||
return self._bucketbackups[bucket.name]
|
||||
return self._bucket_backups[bucket.name]
|
||||
|
||||
@property
|
||||
def all_backups(self): # property
|
||||
def all_backups(self) -> [{}]:
|
||||
if self._backups is None:
|
||||
sys.stderr.write("Enumerating backups")
|
||||
self._backups = {}
|
||||
for bucket in self.backup_buckets:
|
||||
backups_dict = self.get_backups_by_bucket(bucket)
|
||||
for hostname, backups in backups_dict.items():
|
||||
sys.stderr.write('.')
|
||||
sys.stderr.flush()
|
||||
if hostname not in self._backups:
|
||||
self._backups[hostname] = {}
|
||||
self._backups[hostname].update(backups)
|
||||
sys.stderr.write("\n")
|
||||
return self._backups
|
||||
|
||||
def invalidate_host_cache(self, hostname):
|
||||
nuke = []
|
||||
for bucket in self._bucketbackups:
|
||||
if hostname in self._bucketbackups[bucket]:
|
||||
for bucket in self._bucket_backups:
|
||||
if hostname in self._bucket_backups[bucket]:
|
||||
nuke.append(bucket)
|
||||
|
||||
for bucket in nuke:
|
||||
if bucket in self._bucketbackups:
|
||||
del self._bucketbackups[bucket]
|
||||
if bucket in self._bucket_backups:
|
||||
del self._bucket_backups[bucket]
|
||||
self._backups = None
|
||||
|
||||
@property
|
||||
def backups_by_age(self): # property
|
||||
"Returns a dict of {hostname: [(backupnum, age), ...]}"
|
||||
def backups_by_age(self):
|
||||
"""
|
||||
Returns a dict of {hostname: [(backup_num, age), ...]}
|
||||
"""
|
||||
results = defaultdict(list)
|
||||
for hostname, backups in self.all_backups.items():
|
||||
for backupnum, statusdict in backups.items():
|
||||
results[hostname].append((backupnum,
|
||||
statusdict['finalized_age']))
|
||||
for backup_num, statusdict in backups.items():
|
||||
results[hostname].append((backup_num,
|
||||
statusdict["finalized_age"]))
|
||||
return results
|
||||
|
||||
|
||||
def choose_host_to_backup(agedict, target_count=2):
|
||||
"Takes a dict from backups_by_age, returns a hostname to back up."
|
||||
def choose_host_to_backup(age_dict, target_count=2):
|
||||
"""
|
||||
Takes a dict from backups_by_age, returns a hostname to back up.
|
||||
"""
|
||||
|
||||
host_scores = defaultdict(int)
|
||||
|
||||
for hostname, backuplist in agedict.items():
|
||||
bl = sorted(backuplist, key=lambda x: x[1])
|
||||
for hostname, backup_list in age_dict.items():
|
||||
bl = sorted(backup_list, key=lambda x: x[1])
|
||||
if len(bl) > 0 and bl[0][1] == -1:
|
||||
# unfinalized backup alert
|
||||
host_scores[hostname] += 200
|
||||
|
|
@ -199,20 +210,22 @@ def choose_host_to_backup(agedict, target_count=2):
|
|||
host_scores[hostname] -= 100
|
||||
host_scores[hostname] -= len(bl)
|
||||
if len(bl) > 0:
|
||||
# age of oldest backup helps score
|
||||
# age of the oldest backup helps score
|
||||
oldest = bl[0]
|
||||
host_scores[hostname] += log10(oldest[1])
|
||||
# recency of newest backup hurts score
|
||||
# recency of the newest backup hurts score
|
||||
newest = bl[-1]
|
||||
host_scores[hostname] -= log10(max(1, (oldest[1] - newest[1])))
|
||||
|
||||
for candidate, score in sorted(host_scores.items(),
|
||||
key=lambda x: x[1], reverse=True):
|
||||
yield (candidate, score)
|
||||
yield candidate, score
|
||||
|
||||
|
||||
def choose_backups_to_delete(agedict, target_count=2, max_age=30):
|
||||
"Takes a dict from backups_by_age, returns a list of backups to delete"
|
||||
"""
|
||||
Takes a dict from backups_by_age, returns a list of backups to delete
|
||||
"""
|
||||
|
||||
decimate = defaultdict(list)
|
||||
|
||||
|
|
@ -230,279 +243,233 @@ def choose_backups_to_delete(agedict, target_count=2, max_age=30):
|
|||
return decimate
|
||||
|
||||
|
||||
def iter_urls(keyset, expire=86400):
|
||||
"""Given a list of keys and an optional expiration time (in seconds),
|
||||
returns an iterator of URLs to fetch to reassemble the backup."""
|
||||
def make_restore_script(backup: {}, expire=86400) -> str:
|
||||
"""
|
||||
Returns a quick and easy restoration script to restore the given system,
|
||||
requires a backup, and perhaps expire
|
||||
"""
|
||||
|
||||
for key in keyset:
|
||||
yield key.generate_url(expires_in=expire)
|
||||
hostname = backup["hostname"]
|
||||
backup_num = backup["backup_num"]
|
||||
bucket = backup["bucket"]
|
||||
friendly_time = backup["date"].strftime("%Y-%m-%d at %H:%M GMT")
|
||||
|
||||
expire_time = datetime.now() + timedelta(seconds=expire)
|
||||
|
||||
def make_restore_script(backup, expire=86400):
|
||||
"""Returns a quick and easy restoration script to restore the given system,
|
||||
requires a backup, and perhaps expire"""
|
||||
files = [f"'{bucket.get_download_url(i.file_name)}'" for i in backup["files"] + [backup["final_file"]]]
|
||||
|
||||
myhostname = backup['hostname']
|
||||
mybackupnum = backup['backupnum']
|
||||
myfriendlytime = time.strftime('%Y-%m-%d at %H:%M GMT', backup['date'])
|
||||
myexpiretime = time.strftime('%Y-%m-%d at %H:%M GMT',
|
||||
time.gmtime(time.time() + expire))
|
||||
myexpiretimestamp = time.time() + expire
|
||||
output = f"""#!/bin/bash
|
||||
# Restoration script for {hostname} backup {backup_num},
|
||||
# a backup created on {friendly_time}.
|
||||
# To use: bash scriptname /path/to/put/the/files
|
||||
|
||||
output = []
|
||||
# WARNING: THIS FILE EXPIRES AFTER {expire_time.strftime("%Y-%m-%d at %H:%M GMT")}
|
||||
if (( "$(date +%s)" > "{int(expire_time.timestamp() * 1000000)}" )); then
|
||||
echo "Sorry, this restore script is too old."
|
||||
exit 1
|
||||
elif [ -z "$1" ]; then
|
||||
echo "Usage: ./scriptname /path/to/restore/to"
|
||||
exit 1
|
||||
elif [ ! -d "$1" ]; then
|
||||
echo "Target $1 does not exist!"
|
||||
exit 1
|
||||
elif [ -n "$(ls --almost-all "$1")" ]; then
|
||||
echo "Target $1 is not empty!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
output.append('#!/bin/sh\n')
|
||||
output.append('# Restoration script for %s backup %s,\n' % (
|
||||
myhostname, mybackupnum))
|
||||
output.append('# a backup created on %s.\n' % (myfriendlytime))
|
||||
output.append('# To use: bash scriptname /path/to/put/the/files\n\n')
|
||||
output.append('# WARNING: THIS FILE EXPIRES AFTER %s\n' % (myexpiretime))
|
||||
output.append('if [ "`date +%%s`" -gt "%i" ];\n' % (myexpiretimestamp))
|
||||
output.append(' then echo "Sorry, this restore script is too old.";\n')
|
||||
output.append(' exit 1;\n')
|
||||
output.append('fi\n\n')
|
||||
output.append('if [ -z "$1" ];\n')
|
||||
output.append(' then echo "Usage: ./scriptname /path/to/restore/to";\n')
|
||||
output.append(' exit 1;\n')
|
||||
output.append('fi\n\n')
|
||||
output.append('# Check the destination\n')
|
||||
output.append('if [ ! -d $1 ];\n')
|
||||
output.append(' then echo "Target $1 does not exist!";\n')
|
||||
output.append(' exit 1;\n')
|
||||
output.append('fi\n\n')
|
||||
output.append('if [ -n "`ls --almost-all $1`" ];\n')
|
||||
output.append(' then echo "Target $1 is not empty!";\n')
|
||||
output.append(' exit 1;\n')
|
||||
output.append('fi\n\n')
|
||||
output.append('# cd to the destination, create a temporary workspace\n')
|
||||
output.append('cd $1\n')
|
||||
output.append('mkdir .restorescript-scratch\n\n')
|
||||
output.append('# retrieve files\n')
|
||||
# cd to the destination, create a temporary workspace
|
||||
cd "$1"
|
||||
tmp_dir="$i/.restorescript-scratch"
|
||||
mkdir "$tmp_dir"
|
||||
|
||||
mysortedfilelist = []
|
||||
for key in backup['keys']:
|
||||
output.append('wget -O $1/.restorescript-scratch/%s "%s"\n' % (
|
||||
key.name, key.generate_url(expires_in=expire)))
|
||||
mysortedfilelist.append('.restorescript-scratch/' + key.name)
|
||||
mysortedfilelist.sort()
|
||||
files=({' '.join(files)})
|
||||
token='{bucket.get_download_authorization(f'{hostname}.{backup_num}', expire)}'
|
||||
|
||||
output.append('\n# decrypt files\n')
|
||||
output.append('gpg --decrypt-files << EOF\n')
|
||||
output.append('\n'.join(mysortedfilelist))
|
||||
output.append('\nEOF\n')
|
||||
declare –a out_files
|
||||
for i in "${{files[@]}}"; do
|
||||
filename="$(echo "$i" | cut -d/ -f6)"
|
||||
curl "$i" -o "$tmp_dir/$filename" -H "Authorization: $token"
|
||||
if (( $? != 0 )); then
|
||||
echo "Error during download !"
|
||||
exit 1
|
||||
fi
|
||||
out_files+=("$tmp_dir/$filename")
|
||||
done
|
||||
|
||||
output.append('\n# join and untar files\n')
|
||||
output.append('cat .restorescript-scratch/*.tar.?? | tar -xf -\n\n')
|
||||
# decrypt files
|
||||
gpg --decrypt-files "${{out_files[@]}}"
|
||||
|
||||
output.append('echo "DONE! Have a nice day."\n##\n')
|
||||
# join and untar files
|
||||
cat "$tmp_dir/*.tar.??" | tar -xf -
|
||||
|
||||
echo "DONE! Have a nice day."
|
||||
"""
|
||||
return output
|
||||
|
||||
|
||||
def start_archive(hosts):
|
||||
"Starts an archive operation for a list of hosts."
|
||||
if 'LOGNAME' in os.environ:
|
||||
username = os.environ['LOGNAME']
|
||||
"""
|
||||
Starts an archive operation for a list of hosts.
|
||||
"""
|
||||
if "LOGNAME" in environ:
|
||||
username = environ["LOGNAME"]
|
||||
else:
|
||||
try:
|
||||
username = pwd.getpwuid(os.getuid()).pw_name
|
||||
username = getpwuid(getuid()).pw_name
|
||||
except KeyError:
|
||||
username = 'nobody'
|
||||
username = "nobody"
|
||||
|
||||
scriptdir = os.path.dirname(sys.argv[0])
|
||||
|
||||
cmd = [os.path.join(scriptdir, 'BackupPC_archiveStart'), 'archives3',
|
||||
username]
|
||||
cmd = [Path(argv[0]).parents[0] / "BackupPC_archiveStart", "archives3", username]
|
||||
cmd.extend(hosts)
|
||||
|
||||
proc = Popen(cmd)
|
||||
proc.communicate()
|
||||
|
||||
|
||||
def main():
|
||||
# check command line options
|
||||
parser = optparse.OptionParser(
|
||||
usage="usage: %prog [options] [list|delete|script]",
|
||||
description="" +
|
||||
"Companion maintenance script for BackupPC_archiveHost_s3. " +
|
||||
"By default, it assumes the 'list' command, which displays all " +
|
||||
"of the backups currently archived on S3. The 'delete' command " +
|
||||
"is used to delete backups. The 'script' command produces a " +
|
||||
"script that can be used to download and restore a backup.")
|
||||
parser.add_option("-H", "--host", dest="host",
|
||||
help="Name of backed-up host")
|
||||
parser.add_option("-b", "--backup-number", dest="backupnum",
|
||||
help="Backup number")
|
||||
parser.add_option("-a", "--age", dest="age",
|
||||
help="Delete backups older than AGE days")
|
||||
parser.add_option("-k", "--keep", dest="keep",
|
||||
help="When used with --age, keep this many recent " +
|
||||
"backups (default=1)", default=1)
|
||||
parser.add_option("-f", "--filename", dest="filename",
|
||||
help="Output filename for script")
|
||||
parser.add_option("-x", "--expire", dest="expire",
|
||||
help="Maximum age of script, default 86400 seconds")
|
||||
parser.add_option("-t", "--test", dest="test", action="store_true",
|
||||
help="Test mode; don't actually delete")
|
||||
parser.add_option("-u", "--unfinalized", dest="unfinalized",
|
||||
action="store_true", help="Consider unfinalized backups")
|
||||
parser.add_option("-s", "--start-backups", dest="start",
|
||||
action="store_true",
|
||||
help="When used with --age, start backups for hosts " +
|
||||
"with fewer than keep+1 backups")
|
||||
parser.add_option("-l", "--list", dest="list", action="store_true",
|
||||
help="List stored backups after completing operations")
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
bmgr = BackupManager(secrets.accesskey, secrets.sharedkey)
|
||||
|
||||
if options.backupnum and not options.host:
|
||||
parser.error('Must specify --host when specifying --backup-number')
|
||||
|
||||
if options.backupnum:
|
||||
options.backupnum = int(options.backupnum)
|
||||
|
||||
if len(args) == 0:
|
||||
args.append('list')
|
||||
|
||||
if len(args) > 1:
|
||||
parser.error('Too many arguments.')
|
||||
|
||||
if args[0] != 'delete' and options.age:
|
||||
parser.error('--age only makes sense with delete')
|
||||
|
||||
if options.start and not (args[0] == 'delete' and options.age):
|
||||
parser.error('--start-backups only makes sense with delete and --age')
|
||||
|
||||
if args[0] != 'script' and (options.expire or options.filename):
|
||||
parser.error('--expire and --filename only make sense with script')
|
||||
|
||||
if args[0] in ['list', 'script', 'delete']:
|
||||
if options.host:
|
||||
if options.host not in bmgr.all_backups:
|
||||
parser.error('No backups found for host "%s"' % options.host)
|
||||
else:
|
||||
if len(bmgr.all_backups) == 0:
|
||||
parser.error('No buckets found!')
|
||||
else:
|
||||
parser.error('Invalid option: %s' + args[0])
|
||||
|
||||
if args[0] == 'script':
|
||||
if not options.host:
|
||||
parser.error('Must specify --host to generate a script for')
|
||||
|
||||
if not options.backupnum and options.unfinalized:
|
||||
def script(parser: ArgumentParser, bmgr: BackupManager, host: str, unfinalized: bool, backup_num: int = None,
|
||||
expire: int = 86400, filename: str = None):
|
||||
if not backup_num and unfinalized:
|
||||
# assuming highest number
|
||||
options.backupnum = max(bmgr.all_backups[options.host].keys())
|
||||
elif not options.backupnum:
|
||||
backup_num = max(bmgr.all_backups[host].keys())
|
||||
elif not backup_num:
|
||||
# assuming highest finalized number
|
||||
options.backupnum = 0
|
||||
for backup in bmgr.all_backups[options.host].keys():
|
||||
if bmgr.all_backups[options.host][backup]['finalized'] > 0:
|
||||
options.backupnum = max(options.backupnum, backup)
|
||||
if options.backupnum == 0:
|
||||
parser.error('No finalized backups found! Try '
|
||||
'--unfinalized if you dare')
|
||||
backup_num = 0
|
||||
for backup in bmgr.all_backups[host].keys():
|
||||
if bmgr.all_backups[host][backup]["finalized"] > 0:
|
||||
backup_num = max(backup_num, backup)
|
||||
if backup_num == 0:
|
||||
parser.error("No finalized backups found! Try --unfinalized if you dare")
|
||||
|
||||
backup = bmgr.all_backups[options.host][options.backupnum]
|
||||
backup = bmgr.all_backups[host][backup_num]
|
||||
|
||||
if not options.expire:
|
||||
options.expire = "86400"
|
||||
|
||||
if options.filename:
|
||||
fd = open(options.filename, 'w')
|
||||
fd.writelines(make_restore_script(backup,
|
||||
expire=int(options.expire)))
|
||||
if filename:
|
||||
with open(filename, "w") as fd:
|
||||
fd.write(make_restore_script(backup, expire=expire))
|
||||
else:
|
||||
sys.stdout.writelines(make_restore_script(backup,
|
||||
expire=int(options.expire)))
|
||||
elif args[0] == 'delete':
|
||||
to_ignore = int(options.keep)
|
||||
print(make_restore_script(backup, expire=expire))
|
||||
|
||||
|
||||
def delete(bm: BackupManager, keep: int, host: str, backup_num: int, age: int, test: bool,
|
||||
start: bool):
|
||||
to_delete = []
|
||||
if options.host and options.backupnum:
|
||||
print("Will delete backup: %s %i (forced)" % (
|
||||
options.host, options.backupnum))
|
||||
to_delete.append((options.host, options.backupnum))
|
||||
elif options.age:
|
||||
to_delete_dict = choose_backups_to_delete(bmgr.backups_by_age,
|
||||
target_count=to_ignore,
|
||||
max_age=int(options.age))
|
||||
for hostname, backuplist in to_delete_dict.items():
|
||||
for backupstat in backuplist:
|
||||
print("Will delete backup: %s %i (expired at %g days)" % (
|
||||
hostname, backupstat[0], backupstat[1] / 86400.0))
|
||||
to_delete.append((hostname, backupstat[0]))
|
||||
|
||||
if host and backup_num:
|
||||
print(f"Will delete backup: {host} {backup_num} (forced)")
|
||||
to_delete.append((host, backup_num))
|
||||
elif age:
|
||||
to_delete_dict = choose_backups_to_delete(bm.backups_by_age, target_count=keep, max_age=age)
|
||||
for hostname, backup_list in to_delete_dict.items():
|
||||
for backup_stat in backup_list:
|
||||
print(f"Will delete backup: {hostname} {backup_stat[0]} (expired at {backup_stat[1] / 86400.0} days)")
|
||||
to_delete.append((hostname, backup_stat[0]))
|
||||
else:
|
||||
parser.error('Need either an age or a host AND backup number.')
|
||||
return
|
||||
|
||||
if len(to_delete) > 0:
|
||||
for deletehost, deletebackupnum in to_delete:
|
||||
hostbackups = bmgr.all_backups.get(deletehost, {})
|
||||
deletebackup = hostbackups.get(deletebackupnum, {})
|
||||
deletekeys = deletebackup.get('keys', [])
|
||||
finalkey = deletebackup.get('finalkey', None)
|
||||
if len(deletekeys) > 0:
|
||||
sys.stdout.write("Deleting backup: %s %d (%d keys)" % (
|
||||
deletehost, deletebackupnum, len(deletekeys)))
|
||||
for key in deletekeys:
|
||||
if options.test:
|
||||
sys.stdout.write('_')
|
||||
else:
|
||||
key.delete()
|
||||
sys.stdout.write('.')
|
||||
sys.stdout.flush()
|
||||
if finalkey is not None:
|
||||
if options.test:
|
||||
sys.stdout.write('+')
|
||||
else:
|
||||
finalkey.delete()
|
||||
sys.stdout.write('!')
|
||||
sys.stdout.flush()
|
||||
sys.stdout.write('\n')
|
||||
for delete_host, delete_backup_num in to_delete:
|
||||
host_backups = bm.all_backups.get(delete_host, {})
|
||||
delete_backup = host_backups.get(delete_backup_num, {})
|
||||
delete_files = delete_backup.get("files", [])
|
||||
final_file = delete_backup.get("final_file", None)
|
||||
if len(delete_files) > 0:
|
||||
for file in ChargingBar(f"Deleting backup {delete_host} #{delete_backup_num}:", max=len(delete_files)).\
|
||||
iter(delete_files):
|
||||
if not test:
|
||||
file.delete()
|
||||
|
||||
if options.start:
|
||||
for deletehost, deletebackupnum in to_delete:
|
||||
bmgr.invalidate_host_cache(deletehost)
|
||||
score_iter = choose_host_to_backup(bmgr.backups_by_age,
|
||||
target_count=int(options.keep) + 1)
|
||||
if final_file and not test:
|
||||
final_file.delete()
|
||||
|
||||
if start:
|
||||
for delete_host, delete_backup_num in to_delete:
|
||||
bm.invalidate_host_cache(delete_host)
|
||||
score_iter = choose_host_to_backup(bm.backups_by_age, target_count=int(keep) + 1)
|
||||
for candidate, score in score_iter:
|
||||
if score > 0:
|
||||
sys.stdout.write('Starting archive operation for host: '
|
||||
'%s (score=%g)\n' % (candidate, score))
|
||||
print(f"Starting archive operation for host: {candidate} (score={score})")
|
||||
start_archive([candidate])
|
||||
break
|
||||
if args[0] == 'list' or options.list:
|
||||
sys.stdout.write('%25s | %5s | %20s | %5s\n' % (
|
||||
"Hostname", "Bkup#", "Age", "Files"))
|
||||
sys.stdout.write(('-' * 72) + '\n')
|
||||
for hostname, backups in bmgr.all_backups.items():
|
||||
for backupnum in sorted(backups.keys()):
|
||||
filecount = len(backups[backupnum]['keys'])
|
||||
datestruct = backups[backupnum]['date']
|
||||
if backups[backupnum]['finalized'] > 0:
|
||||
inprogress = ''
|
||||
else:
|
||||
inprogress = '*'
|
||||
timestamp = time.mktime(datestruct)
|
||||
delta = int(time.time() - timestamp + time.timezone)
|
||||
if delta < 3600:
|
||||
prettydelta = '%i min ago' % (delta / 60)
|
||||
elif delta < 86400:
|
||||
prettydelta = '%i hr ago' % (delta / 3600)
|
||||
else:
|
||||
days = int(delta / 60 / 60 / 24)
|
||||
if days == 1:
|
||||
s = ''
|
||||
else:
|
||||
s = 's'
|
||||
prettydelta = '%i day%s ago' % (days, s)
|
||||
|
||||
sys.stdout.write('%25s | %5i | %20s | %5i%s\n' % (
|
||||
hostname, backupnum, prettydelta, filecount, inprogress))
|
||||
sys.stdout.write('* == not yet finalized (Age == time of '
|
||||
'last activity)\n')
|
||||
|
||||
if __name__ == '__main__':
|
||||
def list_backups(bm: BackupManager):
|
||||
print(f"{'Hostname':>25} | {'Bkup#':>5} | {'Age':>30} | {'Files':>5}")
|
||||
print(("-" * 72))
|
||||
|
||||
for hostname, backups in bm.all_backups.items():
|
||||
for backup_num in sorted(backups.keys()):
|
||||
filecount = len(backups[backup_num]["files"])
|
||||
date = backups[backup_num]["date"]
|
||||
if backups[backup_num]["finalized"] > 0:
|
||||
in_progress = ""
|
||||
else:
|
||||
in_progress = "*"
|
||||
|
||||
print(f"{hostname:>25} | {backup_num:>5} | {str(datetime.now() - date):>30} | {filecount:>5}{in_progress}")
|
||||
print("* = not yet finalized (Age = time of last activity)")
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser(description="Companion maintenance script for BackupPC_archiveHost_s3. " +
|
||||
"By default, it assumes the 'list' command, which displays all " +
|
||||
"of the backups currently archived on B2. The 'delete' command " +
|
||||
"is used to delete backups. The 'script' command produces a " +
|
||||
"script that can be used to download and restore a backup.")
|
||||
parser.add_argument("-l", "--list", dest="list", action="store_true",
|
||||
help="List stored backups after completing operations")
|
||||
|
||||
subparsers = parser.add_subparsers(required=True, dest="action")
|
||||
subparsers.add_parser("list")
|
||||
|
||||
delete_parser = subparsers.add_parser("delete")
|
||||
delete_parser.add_argument("-s", "--start-backups", dest="start", action="store_true",
|
||||
help="When used with --age, start backups for hosts with fewer than keep+1 backups")
|
||||
delete_parser.add_argument("-k", "--keep", dest="keep", help="When used with --age, keep this many recent backups",
|
||||
default=1)
|
||||
delete_parser.add_argument("-t", "--test", dest="test", action="store_true",
|
||||
help="Test mode; don't actually delete")
|
||||
delete_parser.add_argument("-H", "--host", dest="host", help="Name of backed-up host")
|
||||
delete_parser.add_argument("-b", "--backup-number", dest="backup_num", type=int, help="Backup number")
|
||||
delete_parser.add_argument("-a", "--age", dest="age", help="Delete backups older than AGE days")
|
||||
|
||||
script_parser = subparsers.add_parser("script")
|
||||
script_parser.add_argument("-H", "--host", dest="host", required=True, help="Name of backed-up host")
|
||||
script_parser.add_argument("-b", "--backup-number", dest="backup_num", type=int, help="Backup number")
|
||||
script_parser.add_argument("-f", "--filename", dest="filename", help="Output filename for script")
|
||||
script_parser.add_argument("-x", "--expire", dest="expire", default=86400, help="Maximum age of script")
|
||||
script_parser.add_argument("-u", "--unfinalized", dest="unfinalized", action="store_true",
|
||||
help="Consider unfinalized backups")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
bm = BackupManager(secrets.access_key, secrets.shared_key)
|
||||
|
||||
if args.action == "script" or args.action == "delete":
|
||||
if args.backup_num and not args.host:
|
||||
parser.error("Must specify --host when specifying --backup-number")
|
||||
|
||||
if args.host:
|
||||
if args.host not in bm.all_backups:
|
||||
parser.error(f"No backups found for host \"{args.host}\"")
|
||||
else:
|
||||
if len(bm.all_backups) == 0:
|
||||
parser.error("No buckets found!")
|
||||
|
||||
if args.action == "script":
|
||||
script(parser, bm, args.host, args.backup_num, args.unfinalized, args.expire, args.filename)
|
||||
elif args.action == "delete":
|
||||
if not (args.age or args.host or args.backup_num):
|
||||
parser.error("--age or --host and --backup-number are required")
|
||||
elif args.host and not args.backup_num:
|
||||
parser.error("--backup-number required with --host")
|
||||
elif args.age and (args.host or args.backup_num):
|
||||
parser.error("--age can't be combined with --host or --backup-number")
|
||||
elif args.start and not args.age:
|
||||
parser.error("--start-backups only makes sense with --age")
|
||||
|
||||
delete(bm, args.keep, args.host, args.backup_num, args.age, args.test, args.start)
|
||||
|
||||
if args.action == "list" or args.list:
|
||||
list_backups(bm)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
Loading…
Reference in a new issue