diff --git a/brozzler/worker.py b/brozzler/worker.py index e691f83..a86ce6a 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -162,12 +162,17 @@ class BrozzlerWorker: def ydl_progress(*args, **kwargs): # in case youtube-dl takes a long time, heartbeat site.last_claimed # to prevent another brozzler-worker from claiming the site - if site.rr and doublethink.utcnow() - site.last_claimed > datetime.timedelta(minutes=7): + try: + if site.rr and doublethink.utcnow() - site.last_claimed > datetime.timedelta(minutes=7): + self.logger.debug( + 'heartbeating site.last_claimed to prevent another ' + 'brozzler-worker claiming this site id=%r', site.id) + site.last_claimed = doublethink.utcnow() + site.save() + except: self.logger.debug( - 'heartbeating site.last_claimed to prevent another ' - 'brozzler-worker claiming this site id=%r', site.id) - site.last_claimed = doublethink.utcnow() - site.save() + 'problem heartbeating site.last_claimed site id=%r', + site.id, exc_info=True) ydl_opts = { "outtmpl": "{}/ydl%(autonumber)s.out".format(destdir), diff --git a/setup.py b/setup.py index 8240510..c5c9dbb 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.1b12.dev261', + version='1.1b12.dev262', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt',