flifloo/BackupPC_archive_host_B2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

adding code to indicate when the backup isn't finalized

Browse source
This commit is contained in:
Ryan Tucker 2010-01-04 15:59:34 -05:00
parent 27a0be4b86
commit 1b1ad9445e
2 changed files with 85 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
BackupPC_archiveHost_s3
View file

@ -243,6 +243,8 @@ for i in [tarCreate, compPath, splitPath, parPath]:
# open s3 connection # open s3 connection
bucket = open_s3(secrets.accesskey, secrets.sharedkey, host) bucket = open_s3(secrets.accesskey, secrets.sharedkey, host)
beginning = time.time()
mesg = "Writing archive for host %s, backup #%i" % (host, bkupNum) mesg = "Writing archive for host %s, backup #%i" % (host, bkupNum)
if splitSize > 0 and is_exe(splitPath): if splitSize > 0 and is_exe(splitPath):
mesg += ', split into %i byte chunks' % splitSize mesg += ', split into %i byte chunks' % splitSize
@ -266,6 +268,7 @@ if splitSize > 0 and is_exe(splitPath):
else: else:
fileglob = '%s/%s.%i.tar' % (outLoc, host, bkupNum) fileglob = '%s/%s.%i.tar' % (outLoc, host, bkupNum)
cmd += '> %s' % fileglob cmd += '> %s' % fileglob
filehead = fileglob + '.'
# is there already evidence of this having been done before? # is there already evidence of this having been done before?
if glob.glob(fileglob): if glob.glob(fileglob):
@ -320,3 +323,11 @@ for i in sorted(glob.glob(fileglob)):
bytespersecond = size / (sending_seconds - encrypt_seconds) bytespersecond = size / (sending_seconds - encrypt_seconds)
sys.stdout.write('%s: File sent. Total time %i seconds, crypto time %i seconds, transfer speed %i bytes/second.\n' % (time.strftime('%d-%H:%M:%S'), sending_seconds, encrypt_seconds, bytespersecond)) sys.stdout.write('%s: File sent. Total time %i seconds, crypto time %i seconds, transfer speed %i bytes/second.\n' % (time.strftime('%d-%H:%M:%S'), sending_seconds, encrypt_seconds, bytespersecond))
# finalize the backup
key = MyKey(bucket)
key.key = '%sCOMPLETE' % filehead
key.set_contents_from_string('%s %s "%s"' % (beginning, time.time(), mesg))
key.close()

111
backup-manager.py
View file

@ -34,7 +34,8 @@ def list_backups(bucket):
{'date': Timestamp of backup (int), {'date': Timestamp of backup (int),
'keys': A list of keys comprising the backup, 'keys': A list of keys comprising the backup,
'hostname': Hostname (str), 'hostname': Hostname (str),
'backupnum': Backup number (int) 'backupnum': Backup number (int),
'finalized': 0, or the timestamp the backup was finalized
} }
} }
} }
@ -44,19 +45,24 @@ def list_backups(bucket):
for key in bucket.list(): for key in bucket.list():
keyparts = key.key.split('.') keyparts = key.key.split('.')
encrypted = split = tarred = False encrypted = split = tarred = final = False
if keyparts[-1] == 'gpg': if keyparts[-1] == 'COMPLETE':
encrypted = True final = True
keyparts.pop() keyparts.pop() # back to tar
keyparts.pop() # back to backup number
else:
if keyparts[-1] == 'gpg':
encrypted = True
keyparts.pop()
if keyparts[-1] != 'tar' and len(keyparts[-1]) is 2: if keyparts[-1] != 'tar' and len(keyparts[-1]) is 2:
split = True split = True
keyparts.pop() keyparts.pop()
if keyparts[-1] == 'tar': if keyparts[-1] == 'tar':
tarred = True tarred = True
keyparts.pop() keyparts.pop()
backupnum = int(keyparts.pop()) backupnum = int(keyparts.pop())
hostname = '.'.join(keyparts) hostname = '.'.join(keyparts)
@ -64,13 +70,14 @@ def list_backups(bucket):
lastmod = time.strptime(key.last_modified, '%Y-%m-%dT%H:%M:%S.000Z') lastmod = time.strptime(key.last_modified, '%Y-%m-%dT%H:%M:%S.000Z')
if hostname in backups.keys(): if hostname in backups.keys():
if backupnum in backups[hostname].keys(): if not backupnum in backups[hostname].keys():
backups[hostname][backupnum]['keys'].append(key) backups[hostname][backupnum] = {'date': lastmod, 'hostname': hostname, 'backupnum': backupnum, 'finalized': 0, 'keys': []}
else:
backups[hostname][backupnum] = {'keys': [key], 'date': lastmod, 'hostname': hostname, 'backupnum': backupnum}
else: else:
backups[hostname] = {backupnum: {'keys': [key], 'date': lastmod, 'hostname': hostname, 'backupnum': backupnum}} backups[hostname] = {backupnum: {'date': lastmod, 'hostname': hostname, 'backupnum': backupnum, 'finalized': 0, 'keys': []}}
if final:
backups[hostname][backupnum]['finalized'] = lastmod
else:
backups[hostname][backupnum]['keys'].append(key)
return backups return backups
def iter_urls(keyset, expire=86400): def iter_urls(keyset, expire=86400):
@ -153,6 +160,8 @@ def main():
help="Maximum age of script, default 86400 seconds") help="Maximum age of script, default 86400 seconds")
parser.add_option("-t", "--test", dest="test", action="store_true", parser.add_option("-t", "--test", dest="test", action="store_true",
help="Test mode; don't actually delete") help="Test mode; don't actually delete")
parser.add_option("-u", "--unfinalized", dest="unfinalized",
action="store_true", help="Consider unfinalized backups")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
@ -194,9 +203,17 @@ def main():
backups = list_backups(buckets.next()) backups = list_backups(buckets.next())
if not options.backupnum: if not options.backupnum and options.unfinalized:
# assuming highest number # assuming highest number
options.backupnum = max(backups[options.host].keys()) options.backupnum = max(backups[options.host].keys())
elif not options.backupnum:
# assuming highest finalized number
options.backupnum = 0
for backup in backups[options.host].keys():
if backups[options.host][backup]['finalized'] > 0:
options.backupnum = max(options.backupnum, backup)
if options.backupnum == 0:
parser.error('No finalized backups found! Try --unfinalized if you dare')
backup = backups[options.host][options.backupnum] backup = backups[options.host][options.backupnum]
@ -217,7 +234,12 @@ def main():
backups = hostnames[hostname] backups = hostnames[hostname]
for backupnum in backups.keys(): for backupnum in backups.keys():
filecount = len(backups[backupnum]['keys']) filecount = len(backups[backupnum]['keys'])
datestruct = backups[backupnum]['date'] if backups[backupnum]['finalized'] > 0:
datestruct = backups[backupnum]['finalized']
inprogress = ''
else:
datestruct = backups[backupnum]['date']
inprogress = '*'
timestamp = time.mktime(datestruct) timestamp = time.mktime(datestruct)
delta = int(time.time() - timestamp) delta = int(time.time() - timestamp)
if delta < 3600: if delta < 3600:
@ -232,7 +254,8 @@ def main():
s = 's' s = 's'
prettydelta = '%i day%s ago' % (days, s) prettydelta = '%i day%s ago' % (days, s)
sys.stdout.write('%25s | %5i | %20s | %5i\n' % (hostname, backupnum, prettydelta, filecount)) sys.stdout.write('%25s | %5i | %20s | %5i%s\n' % (hostname, backupnum, prettydelta, filecount, inprogress))
sys.stdout.write('* == not yet finalized (Age == time of last activity)\n')
elif args[0] == 'delete': elif args[0] == 'delete':
if options.age: if options.age:
maxage = int(options.age)*86400 maxage = int(options.age)*86400
@ -242,18 +265,24 @@ def main():
backups = hostnames[hostname] backups = hostnames[hostname]
for backupnum in backups.keys(): for backupnum in backups.keys():
filecount = len(backups[backupnum]['keys']) filecount = len(backups[backupnum]['keys'])
datestruct = backups[backupnum]['date'] if backups[backupnum]['finalized'] > 0:
datestruct = backups[backupnum]['finalized']
else:
datestruct = backups[backupnum]['date']
timestamp = time.mktime(datestruct) timestamp = time.mktime(datestruct)
delta = int(time.time() - timestamp) delta = int(time.time() - timestamp)
if delta > maxage: if delta > maxage:
sys.stdout.write('Deleting %s #%i (%i files, age %.2f days)...' % (hostname, backupnum, filecount, delta/86400.0)) if options.unfinalized and backups[backupnum]['finalized'] > 0:
for key in backups[backupnum]['keys']: sys.stdout.write('Bypassing finalized backup %s #%i (%i files, age %.2f days)\n' % (hostname, backupnum, filecount, delta/86400.0))
if options.test: else:
sys.stdout.write('*') sys.stdout.write('Deleting %s #%i (%i files, age %.2f days)...' % (hostname, backupnum, filecount, delta/86400.0))
else: for key in backups[backupnum]['keys']:
key.delete() if options.test:
sys.stdout.write('.') sys.stdout.write('*')
sys.stdout.write('\n') else:
key.delete()
sys.stdout.write('.')
sys.stdout.write('\n')
elif options.host and options.backupnum: elif options.host and options.backupnum:
for bucket in buckets: for bucket in buckets:
hostnames = list_backups(bucket) hostnames = list_backups(bucket)
@ -262,18 +291,26 @@ def main():
parser.error('Backup number %i not found' % options.backupnum) parser.error('Backup number %i not found' % options.backupnum)
toast = hostnames[options.host][options.backupnum] toast = hostnames[options.host][options.backupnum]
filecount = len(toast['keys']) filecount = len(toast['keys'])
if toast['finalized'] > 0:
datestruct = toast['finalized']
else:
datestruct = toast['date']
datestruct = toast['date'] datestruct = toast['date']
timestamp = time.mktime(datestruct) timestamp = time.mktime(datestruct)
delta = int(time.time() - timestamp) delta = int(time.time() - timestamp)
sys.stdout.write('Deleting %s #%i (%i files, age %.2f days)...' % (options.host, options.backupnum, filecount, delta/86400.0)) if options.unfinalized and toast['finalized'] > 0:
for key in toast['keys']: sys.stdout.write('Bypassing finalized backup %s #%i (%i files, age %.2f days)\n' % (hostname, backupnum, filecount, delta/86400.0))
if options.test: else:
sys.stdout.write('*') sys.stdout.write('Deleting %s #%i (%i files, age %.2f days)...' % (options.host, options.backupnum, filecount, delta/86400.0))
else: for key in toast['keys']:
key.delete() if options.test:
sys.stdout.write('.') sys.stdout.write('*')
sys.stdout.write('\n') else:
key.delete()
sys.stdout.write('.')
sys.stdout.write('\n')
else: else:
parser.error('Host %s not found' % options.host) parser.error('Host %s not found' % options.host)
else: else: