Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-20 12:54:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

handle another rethinkdb outage corner case

Browse source
This commit is contained in:
Noah Levitt 2017-05-01 14:12:43 -07:00
parent 389db01458
commit 9d4cbbf6eb
2 changed files with 6 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

5
brozzler/worker.py
View file

@ -35,6 +35,7 @@ import doublethink
import tempfile import tempfile
import urlcanon import urlcanon
from requests.structures import CaseInsensitiveDict from requests.structures import CaseInsensitiveDict
import rethinkdb as r
class ExtraHeaderAdder(urllib.request.BaseHandler): class ExtraHeaderAdder(urllib.request.BaseHandler):
def __init__(self, extra_headers): def __init__(self, extra_headers):
@ -546,6 +547,10 @@ class BrozzlerWorker:
pass pass
time.sleep(0.5) time.sleep(0.5)
self.logger.info("shutdown requested") self.logger.info("shutdown requested")
except r.ReqlError as e:
self.logger.error(
"caught rethinkdb exception, will try to proceed",
exc_info=True)
except brozzler.ShutdownRequested: except brozzler.ShutdownRequested:
self.logger.info("shutdown requested") self.logger.info("shutdown requested")
except: except:

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b11.dev242', version='1.1b11.dev243',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',