trapping socket.error; retry old files on a restart instead of starting anew

This commit is contained in:
Ryan Tucker 2010-01-09 15:00:51 -05:00
parent 2581d85058
commit 55e1a5cb4a

View file

@ -17,6 +17,7 @@ import glob
import md5
import os
import secrets
import socket
import sys
import time
@ -280,8 +281,9 @@ else:
filehead = fileglob + '.'
# is there already evidence of this having been done before?
if glob.glob(fileglob):
if glob.glob('%s/%s.*.tar.*' % (outLoc, host)):
logging.info('Evidence of failed execution run prior! Finishing it.')
fileglob = '%s/%s.*.tar.*' % (outLoc, host)
else:
logging.debug('Executing %s' % cmd)
@ -315,10 +317,13 @@ for i in sorted(glob.glob(fileglob)):
key.set_acl('private')
key.close()
retry_count = max_retries+1
except boto.exception.S3ResponseError as e:
except boto.exception.S3ResponseError, socket.error as e:
retry_count += 1
sleeptime = 2**retry_count
logger.error('Encountered S3 exception %s, retrying in %i seconds (%i/%i)' % (e, sleeptime, retry_count, max_retries))
err = 'Encountered exception %s, retrying in %i seconds (%i/%i)' % (e, sleeptime, retry_count, max_retries)
logger.error(err)
sys.stdout.write(time.strftime('%d-%H:%M:%S') + ': ' + err + '\n')
sys.stdout.flush()
time.sleep(sleeptime)
size = os.path.getsize(sendfile)