have _warcprox_write_record also raise ProxyError when appropriate, and test this

This commit is contained in:
Noah Levitt 2017-04-18 16:58:51 -07:00
parent ac972d399f
commit 5603ff5380
3 changed files with 12 additions and 1 deletions

View File

@ -205,6 +205,8 @@ class BrozzlerWorker:
'got "%s %s" response on warcprox '
'WARCPROX_WRITE_RECORD request (expected 204)',
e.getcode(), e.info())
except urllib.error.URLError as e:
raise brozzler.ProxyError('_warcprox_write_record: %s', e)
def _remember_videos(self, page, ydl_spy):
if not 'videos' in page:

View File

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup(
name='brozzler',
version='1.1b11.dev234',
version='1.1b11.dev235',
description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler',
author='Noah Levitt',

View File

@ -146,6 +146,15 @@ def test_proxy_down():
with pytest.raises(brozzler.ProxyError):
worker._fetch_url(site, page)
# WARCPROX_WRITE_RECORD
with pytest.raises(brozzler.ProxyError):
worker._warcprox_write_record(
warcprox_address=not_listening_proxy,
url='test://proxy_down/warcprox_write_record',
warc_type='metadata',
content_type='text/plain',
payload=b'''payload doesn't matter here''')
def test_start_stop_backwards_compat():
site = brozzler.Site(None, {'seed': 'http://example.com/'})
assert len(site.starts_and_stops) == 1