diff --git a/brozzler/worker.py b/brozzler/worker.py index 45739b7..6623ed0 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -208,7 +208,11 @@ class BrozzlerWorker: 'WARCPROX_WRITE_RECORD request (expected 204)', e.getcode(), e.info()) except urllib.error.URLError as e: - raise brozzler.ProxyError('_warcprox_write_record: %s', e) + raise brozzler.ProxyError( + 'proxy error on WARCPROX_WRITE_RECORD %s' % url) from e + except ConnectionError as e: + raise brozzler.ProxyError( + 'proxy error on WARCPROX_WRITE_RECORD %s' % url) from e def _remember_videos(self, page, ydl_spy): if not 'videos' in page: @@ -266,7 +270,8 @@ class BrozzlerWorker: and self._proxy_for(site)): # connection problem when using a proxy == proxy error (XXX?) raise brozzler.ProxyError( - 'youtube-dl hit apparent proxy error', e) + 'youtube-dl hit apparent proxy error from ' + '%s' % page.url) from e else: raise @@ -399,7 +404,8 @@ class BrozzlerWorker: page.url, proxies=proxies, headers=site.extra_headers(), verify=False) except requests.exceptions.ProxyError as e: - raise brozzler.ProxyError(e) + raise brozzler.ProxyError( + 'proxy error fetching %s' % page.url) from e def _needs_browsing(self, page, brozzler_spy): final_bounces = brozzler_spy.final_bounces(page.url) diff --git a/setup.py b/setup.py index 85b8efc..2ad0800 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.1b11.dev244', + version='1.1b11.dev245', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt',