Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 00:29:53 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

recognize ConnectionError (of which ConnectionResetError is a subclass) in _warcprox_write_record as a proxy error

Browse Source
This commit is contained in:
Noah Levitt 2017-05-12 10:03:53 -07:00
parent 054625b8a5
commit 60c5a7c1c4
2 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
brozzler/worker.py
View File

@ -208,7 +208,11 @@ class BrozzlerWorker:
'WARCPROX_WRITE_RECORD request (expected 204)', 'WARCPROX_WRITE_RECORD request (expected 204)',
e.getcode(), e.info()) e.getcode(), e.info())
except urllib.error.URLError as e: except urllib.error.URLError as e:
raise brozzler.ProxyError('_warcprox_write_record: %s', e) raise brozzler.ProxyError(
'proxy error on WARCPROX_WRITE_RECORD %s' % url) from e
except ConnectionError as e:
raise brozzler.ProxyError(
'proxy error on WARCPROX_WRITE_RECORD %s' % url) from e
def _remember_videos(self, page, ydl_spy): def _remember_videos(self, page, ydl_spy):
if not 'videos' in page: if not 'videos' in page:
@ -266,7 +270,8 @@ class BrozzlerWorker:
and self._proxy_for(site)): and self._proxy_for(site)):
# connection problem when using a proxy == proxy error (XXX?) # connection problem when using a proxy == proxy error (XXX?)
raise brozzler.ProxyError( raise brozzler.ProxyError(
'youtube-dl hit apparent proxy error', e) 'youtube-dl hit apparent proxy error from '
'%s' % page.url) from e
else: else:
raise raise
@ -399,7 +404,8 @@ class BrozzlerWorker:
page.url, proxies=proxies, headers=site.extra_headers(), page.url, proxies=proxies, headers=site.extra_headers(),
verify=False) verify=False)
except requests.exceptions.ProxyError as e: except requests.exceptions.ProxyError as e:
raise brozzler.ProxyError(e) raise brozzler.ProxyError(
'proxy error fetching %s' % page.url) from e
def _needs_browsing(self, page, brozzler_spy): def _needs_browsing(self, page, brozzler_spy):
final_bounces = brozzler_spy.final_bounces(page.url) final_bounces = brozzler_spy.final_bounces(page.url)

2
setup.py
View File

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b11.dev244', version='1.1b11.dev245',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',