flifloo/BackupPC_archive_host_B2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

pyflakes / PEP8 compliance

Browse source
This commit is contained in:
Ryan Tucker 2013-02-06 19:00:32 -05:00
parent 4f4574b727
commit e9ee689844
1 changed files with 86 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

119
backup-manager.py
View file

@ -2,7 +2,7 @@
# #
# Script to manage S3-stored backups # Script to manage S3-stored backups
# #
# Copyright (c) 2009-2011 Ryan S. Tucker # Copyright (c) 2009-2013 Ryan S. Tucker
# #
# Permission is hereby granted, free of charge, to any person obtaining a copy # Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal # of this software and associated documentation files (the "Software"), to deal
@ -30,14 +30,14 @@ import sys
import time import time
from boto.s3.connection import S3Connection from boto.s3.connection import S3Connection
from boto.s3.key import Key
import boto.exception
from collections import defaultdict from collections import defaultdict
from math import log10 from math import log10
from subprocess import * from subprocess import Popen
class BackupManager: class BackupManager:
def __init__(self, accesskey, sharedkey): def __init__(self, accesskey, sharedkey):
self._accesskey = accesskey self._accesskey = accesskey
self._connection = S3Connection(accesskey, sharedkey) self._connection = S3Connection(accesskey, sharedkey)
@ -79,7 +79,7 @@ class BackupManager:
for key in bucket.list(): for key in bucket.list():
keyparts = key.key.split('.') keyparts = key.key.split('.')
encrypted = split = tarred = final = False final = False
if keyparts[-1] == 'COMPLETE': if keyparts[-1] == 'COMPLETE':
final = True final = True
@ -87,15 +87,12 @@ class BackupManager:
keyparts.pop() # back to backup number keyparts.pop() # back to backup number
else: else:
if keyparts[-1] == 'gpg': if keyparts[-1] == 'gpg':
encrypted = True
keyparts.pop() keyparts.pop()
if keyparts[-1] != 'tar' and len(keyparts[-1]) is 2: if keyparts[-1] != 'tar' and len(keyparts[-1]) is 2:
split = True
keyparts.pop() keyparts.pop()
if keyparts[-1] == 'tar': if keyparts[-1] == 'tar':
tarred = True
keyparts.pop() keyparts.pop()
nextpart = keyparts.pop() nextpart = keyparts.pop()
@ -105,13 +102,32 @@ class BackupManager:
backupnum = int(nextpart) backupnum = int(nextpart)
hostname = '.'.join(keyparts) hostname = '.'.join(keyparts)
lastmod = time.strptime(key.last_modified, '%Y-%m-%dT%H:%M:%S.000Z') lastmod = time.strptime(key.last_modified,
'%Y-%m-%dT%H:%M:%S.000Z')
if hostname in backups.keys(): if hostname in backups.keys():
if not backupnum in backups[hostname].keys(): if not backupnum in backups[hostname].keys():
backups[hostname][backupnum] = {'date': lastmod, 'hostname': hostname, 'backupnum': backupnum, 'finalized': 0, 'keys': [], 'finalkey': None, 'finalized_age': -1} backups[hostname][backupnum] = {
'date': lastmod,
'hostname': hostname,
'backupnum': backupnum,
'finalized': 0,
'keys': [],
'finalkey': None,
'finalized_age': -1,
}
else: else:
backups[hostname] = {backupnum: {'date': lastmod, 'hostname': hostname, 'backupnum': backupnum, 'finalized': 0, 'keys': [], 'finalkey': None, 'finalized_age': -1}} backups[hostname] = {
backupnum: {
'date': lastmod,
'hostname': hostname,
'backupnum': backupnum,
'finalized': 0,
'keys': [],
'finalkey': None,
'finalized_age': -1,
}
}
if final: if final:
backups[hostname][backupnum]['finalized'] = lastmod backups[hostname][backupnum]['finalized'] = lastmod
backups[hostname][backupnum]['finalkey'] = key backups[hostname][backupnum]['finalkey'] = key
@ -136,7 +152,8 @@ class BackupManager:
sys.stderr.write("Enumerating backups") sys.stderr.write("Enumerating backups")
self._backups = {} self._backups = {}
for bucket in self.backup_buckets: for bucket in self.backup_buckets:
for hostname, backups in self.get_backups_by_bucket(bucket).items(): backups_dict = self.get_backups_by_bucket(bucket)
for hostname, backups in backups_dict.items():
sys.stderr.write('.') sys.stderr.write('.')
sys.stderr.flush() sys.stderr.flush()
if hostname not in self._backups: if hostname not in self._backups:
@ -162,9 +179,11 @@ class BackupManager:
results = defaultdict(list) results = defaultdict(list)
for hostname, backups in self.all_backups.items(): for hostname, backups in self.all_backups.items():
for backupnum, statusdict in backups.items(): for backupnum, statusdict in backups.items():
results[hostname].append((backupnum, statusdict['finalized_age'])) results[hostname].append((backupnum,
statusdict['finalized_age']))
return results return results
def choose_host_to_backup(agedict, target_count=2): def choose_host_to_backup(agedict, target_count=2):
"Takes a dict from backups_by_age, returns a hostname to back up." "Takes a dict from backups_by_age, returns a hostname to back up."
@ -187,9 +206,11 @@ def choose_host_to_backup(agedict, target_count=2):
newest = bl[-1] newest = bl[-1]
host_scores[hostname] -= log10(max(1, (oldest[1] - newest[1]))) host_scores[hostname] -= log10(max(1, (oldest[1] - newest[1])))
for candidate, score in sorted(host_scores.items(), key=lambda x: x[1], reverse=True): for candidate, score in sorted(host_scores.items(),
key=lambda x: x[1], reverse=True):
yield (candidate, score) yield (candidate, score)
def choose_backups_to_delete(agedict, target_count=2, max_age=30): def choose_backups_to_delete(agedict, target_count=2, max_age=30):
"Takes a dict from backups_by_age, returns a list of backups to delete" "Takes a dict from backups_by_age, returns a list of backups to delete"
@ -203,11 +224,12 @@ def choose_backups_to_delete(agedict, target_count=2, max_age=30):
while len(bl) > target_count: while len(bl) > target_count:
backup = bl.pop() backup = bl.pop()
if backup[1] > max_age*24*60*60: if backup[1] > (max_age * 24 * 60 * 60):
decimate[hostname].append(backup) decimate[hostname].append(backup)
return decimate return decimate
def iter_urls(keyset, expire=86400): def iter_urls(keyset, expire=86400):
"""Given a list of keys and an optional expiration time (in seconds), """Given a list of keys and an optional expiration time (in seconds),
returns an iterator of URLs to fetch to reassemble the backup.""" returns an iterator of URLs to fetch to reassemble the backup."""
@ -215,26 +237,28 @@ def iter_urls(keyset, expire=86400):
for key in keyset: for key in keyset:
yield key.generate_url(expires_in=expire) yield key.generate_url(expires_in=expire)
def make_restore_script(backup, expire=86400): def make_restore_script(backup, expire=86400):
"""Returns a quick and easy restoration script to restore the given system, """Returns a quick and easy restoration script to restore the given system,
requires a backup, and perhaps expire""" requires a backup, and perhaps expire"""
myhostname = backup['hostname'] myhostname = backup['hostname']
mybackupnum = backup['backupnum'] mybackupnum = backup['backupnum']
myfilecount = len(backup['keys'])
myfriendlytime = time.strftime('%Y-%m-%d at %H:%M GMT', backup['date']) myfriendlytime = time.strftime('%Y-%m-%d at %H:%M GMT', backup['date'])
myexpiretime = time.strftime('%Y-%m-%d at %H:%M GMT', time.gmtime(time.time()+expire)) myexpiretime = time.strftime('%Y-%m-%d at %H:%M GMT',
time.gmtime(time.time() + expire))
myexpiretimestamp = time.time() + expire myexpiretimestamp = time.time() + expire
output = [] output = []
output.append('#!/bin/sh\n') output.append('#!/bin/sh\n')
output.append('# Restoration script for %s backup %s,\n' % (myhostname, mybackupnum)) output.append('# Restoration script for %s backup %s,\n' % (
myhostname, mybackupnum))
output.append('# a backup created on %s.\n' % (myfriendlytime)) output.append('# a backup created on %s.\n' % (myfriendlytime))
output.append('# To use: bash scriptname /path/to/put/the/files\n\n') output.append('# To use: bash scriptname /path/to/put/the/files\n\n')
output.append('# WARNING: THIS FILE EXPIRES AFTER %s\n' % (myexpiretime)) output.append('# WARNING: THIS FILE EXPIRES AFTER %s\n' % (myexpiretime))
output.append('if [ "`date +%%s`" -gt "%i" ];\n' % (myexpiretimestamp)) output.append('if [ "`date +%%s`" -gt "%i" ];\n' % (myexpiretimestamp))
output.append(' then echo "Sorry, but this restore script is too old.";\n') output.append(' then echo "Sorry, this restore script is too old.";\n')
output.append(' exit 1;\n') output.append(' exit 1;\n')
output.append('fi\n\n') output.append('fi\n\n')
output.append('if [ -z "$1" ];\n') output.append('if [ -z "$1" ];\n')
@ -257,7 +281,8 @@ def make_restore_script(backup, expire=86400):
mysortedfilelist = [] mysortedfilelist = []
for key in backup['keys']: for key in backup['keys']:
output.append('wget -O $1/.restorescript-scratch/%s "%s"\n' % (key.name, key.generate_url(expires_in=expire))) output.append('wget -O $1/.restorescript-scratch/%s "%s"\n' % (
key.name, key.generate_url(expires_in=expire)))
mysortedfilelist.append('.restorescript-scratch/' + key.name) mysortedfilelist.append('.restorescript-scratch/' + key.name)
mysortedfilelist.sort() mysortedfilelist.sort()
@ -273,6 +298,7 @@ def make_restore_script(backup, expire=86400):
return output return output
def start_archive(hosts): def start_archive(hosts):
"Starts an archive operation for a list of hosts." "Starts an archive operation for a list of hosts."
if 'LOGNAME' in os.environ: if 'LOGNAME' in os.environ:
@ -292,6 +318,7 @@ def start_archive(hosts):
proc = Popen(cmd) proc = Popen(cmd)
proc.communicate() proc.communicate()
def main(): def main():
# check command line options # check command line options
parser = optparse.OptionParser( parser = optparse.OptionParser(
@ -301,8 +328,7 @@ def main():
"By default, it assumes the 'list' command, which displays all " + "By default, it assumes the 'list' command, which displays all " +
"of the backups currently archived on S3. The 'delete' command " + "of the backups currently archived on S3. The 'delete' command " +
"is used to delete backups. The 'script' command produces a " + "is used to delete backups. The 'script' command produces a " +
"script that can be used to download and restore a backup." "script that can be used to download and restore a backup.")
)
parser.add_option("-H", "--host", dest="host", parser.add_option("-H", "--host", dest="host",
help="Name of backed-up host") help="Name of backed-up host")
parser.add_option("-b", "--backup-number", dest="backupnum", parser.add_option("-b", "--backup-number", dest="backupnum",
@ -310,7 +336,8 @@ def main():
parser.add_option("-a", "--age", dest="age", parser.add_option("-a", "--age", dest="age",
help="Delete backups older than AGE days") help="Delete backups older than AGE days")
parser.add_option("-k", "--keep", dest="keep", parser.add_option("-k", "--keep", dest="keep",
help="When used with --age, keep this many recent backups (default=1)", default=1) help="When used with --age, keep this many recent " +
"backups (default=1)", default=1)
parser.add_option("-f", "--filename", dest="filename", parser.add_option("-f", "--filename", dest="filename",
help="Output filename for script") help="Output filename for script")
parser.add_option("-x", "--expire", dest="expire", parser.add_option("-x", "--expire", dest="expire",
@ -320,8 +347,11 @@ def main():
parser.add_option("-u", "--unfinalized", dest="unfinalized", parser.add_option("-u", "--unfinalized", dest="unfinalized",
action="store_true", help="Consider unfinalized backups") action="store_true", help="Consider unfinalized backups")
parser.add_option("-s", "--start-backups", dest="start", parser.add_option("-s", "--start-backups", dest="start",
action="store_true", help="When used with --age, start backups for hosts with fewer than keep+1 backups") action="store_true",
parser.add_option("-l", "--list", dest="list", action="store_true", help="List stored backups after completing operations") help="When used with --age, start backups for hosts " +
"with fewer than keep+1 backups")
parser.add_option("-l", "--list", dest="list", action="store_true",
help="List stored backups after completing operations")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
@ -372,7 +402,8 @@ def main():
if bmgr.all_backups[options.host][backup]['finalized'] > 0: if bmgr.all_backups[options.host][backup]['finalized'] > 0:
options.backupnum = max(options.backupnum, backup) options.backupnum = max(options.backupnum, backup)
if options.backupnum == 0: if options.backupnum == 0:
parser.error('No finalized backups found! Try --unfinalized if you dare') parser.error('No finalized backups found! Try '
'--unfinalized if you dare')
backup = bmgr.all_backups[options.host][options.backupnum] backup = bmgr.all_backups[options.host][options.backupnum]
@ -381,20 +412,26 @@ def main():
if options.filename: if options.filename:
fd = open(options.filename, 'w') fd = open(options.filename, 'w')
fd.writelines(make_restore_script(backup, expire=int(options.expire))) fd.writelines(make_restore_script(backup,
expire=int(options.expire)))
else: else:
sys.stdout.writelines(make_restore_script(backup, expire=int(options.expire))) sys.stdout.writelines(make_restore_script(backup,
expire=int(options.expire)))
elif args[0] == 'delete': elif args[0] == 'delete':
to_ignore = int(options.keep) to_ignore = int(options.keep)
to_delete = [] to_delete = []
if options.host and options.backupnum: if options.host and options.backupnum:
print("Will delete backup: %s %i (forced)" % (options.host, options.backupnum)) print("Will delete backup: %s %i (forced)" % (
options.host, options.backupnum))
to_delete.append((options.host, options.backupnum)) to_delete.append((options.host, options.backupnum))
elif options.age: elif options.age:
to_delete_dict = choose_backups_to_delete(bmgr.backups_by_age, target_count=to_ignore, max_age=int(options.age)) to_delete_dict = choose_backups_to_delete(bmgr.backups_by_age,
target_count=to_ignore,
max_age=int(options.age))
for hostname, backuplist in to_delete_dict.items(): for hostname, backuplist in to_delete_dict.items():
for backupstat in backuplist: for backupstat in backuplist:
print("Will delete backup: %s %i (expired, age=%g days)" % (hostname, backupstat[0], backupstat[1]/86400.0)) print("Will delete backup: %s %i (expired at %g days)" % (
hostname, backupstat[0], backupstat[1] / 86400.0))
to_delete.append((hostname, backupstat[0])) to_delete.append((hostname, backupstat[0]))
else: else:
@ -407,7 +444,8 @@ def main():
deletekeys = deletebackup.get('keys', []) deletekeys = deletebackup.get('keys', [])
finalkey = deletebackup.get('finalkey', None) finalkey = deletebackup.get('finalkey', None)
if len(deletekeys) > 0: if len(deletekeys) > 0:
sys.stdout.write("Deleting backup: %s %d (%d keys)" % (deletehost, deletebackupnum, len(deletekeys))) sys.stdout.write("Deleting backup: %s %d (%d keys)" % (
deletehost, deletebackupnum, len(deletekeys)))
for key in deletekeys: for key in deletekeys:
if options.test: if options.test:
sys.stdout.write('_') sys.stdout.write('_')
@ -427,15 +465,18 @@ def main():
if options.start: if options.start:
for deletehost, deletebackupnum in to_delete: for deletehost, deletebackupnum in to_delete:
bmgr.invalidate_host_cache(deletehost) bmgr.invalidate_host_cache(deletehost)
score_iter = choose_host_to_backup(bmgr.backups_by_age, target_count=int(options.keep)+1) score_iter = choose_host_to_backup(bmgr.backups_by_age,
target_count=int(options.keep) + 1)
for candidate, score in score_iter: for candidate, score in score_iter:
if score > 0: if score > 0:
sys.stdout.write('Starting archive operation for host: %s (score=%g)\n' % (candidate, score)) sys.stdout.write('Starting archive operation for host: '
'%s (score=%g)\n' % (candidate, score))
start_archive([candidate]) start_archive([candidate])
break break
if args[0] == 'list' or options.list: if args[0] == 'list' or options.list:
sys.stdout.write('%25s | %5s | %20s | %5s\n' % ("Hostname", "Bkup#", "Age", "Files")) sys.stdout.write('%25s | %5s | %20s | %5s\n' % (
sys.stdout.write('-'*72 + '\n') "Hostname", "Bkup#", "Age", "Files"))
sys.stdout.write(('-' * 72) + '\n')
for hostname, backups in bmgr.all_backups.items(): for hostname, backups in bmgr.all_backups.items():
for backupnum in sorted(backups.keys()): for backupnum in sorted(backups.keys()):
filecount = len(backups[backupnum]['keys']) filecount = len(backups[backupnum]['keys'])
@ -458,8 +499,10 @@ def main():
s = 's' s = 's'
prettydelta = '%i day%s ago' % (days, s) prettydelta = '%i day%s ago' % (days, s)
sys.stdout.write('%25s | %5i | %20s | %5i%s\n' % (hostname, backupnum, prettydelta, filecount, inprogress)) sys.stdout.write('%25s | %5i | %20s | %5i%s\n' % (
sys.stdout.write('* == not yet finalized (Age == time of last activity)\n') hostname, backupnum, prettydelta, filecount, inprogress))
sys.stdout.write('* == not yet finalized (Age == time of '
'last activity)\n')
if __name__ == '__main__': if __name__ == '__main__':
main() main()