diff --git a/brozzler/worker.py b/brozzler/worker.py index 1aef258..1c65036 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -205,6 +205,8 @@ class BrozzlerWorker: 'got "%s %s" response on warcprox ' 'WARCPROX_WRITE_RECORD request (expected 204)', e.getcode(), e.info()) + except urllib.error.URLError as e: + raise brozzler.ProxyError('_warcprox_write_record: %s', e) def _remember_videos(self, page, ydl_spy): if not 'videos' in page: diff --git a/setup.py b/setup.py index fb5b62a..86c72c6 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.1b11.dev234', + version='1.1b11.dev235', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt', diff --git a/tests/test_units.py b/tests/test_units.py index d9825d5..911fceb 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -146,6 +146,15 @@ def test_proxy_down(): with pytest.raises(brozzler.ProxyError): worker._fetch_url(site, page) + # WARCPROX_WRITE_RECORD + with pytest.raises(brozzler.ProxyError): + worker._warcprox_write_record( + warcprox_address=not_listening_proxy, + url='test://proxy_down/warcprox_write_record', + warc_type='metadata', + content_type='text/plain', + payload=b'''payload doesn't matter here''') + def test_start_stop_backwards_compat(): site = brozzler.Site(None, {'seed': 'http://example.com/'}) assert len(site.starts_and_stops) == 1