mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-04-19 15:25:59 -04:00
have _warcprox_write_record also raise ProxyError when appropriate, and test this
This commit is contained in:
parent
ac972d399f
commit
5603ff5380
@ -205,6 +205,8 @@ class BrozzlerWorker:
|
||||
'got "%s %s" response on warcprox '
|
||||
'WARCPROX_WRITE_RECORD request (expected 204)',
|
||||
e.getcode(), e.info())
|
||||
except urllib.error.URLError as e:
|
||||
raise brozzler.ProxyError('_warcprox_write_record: %s', e)
|
||||
|
||||
def _remember_videos(self, page, ydl_spy):
|
||||
if not 'videos' in page:
|
||||
|
2
setup.py
2
setup.py
@ -32,7 +32,7 @@ def find_package_data(package):
|
||||
|
||||
setuptools.setup(
|
||||
name='brozzler',
|
||||
version='1.1b11.dev234',
|
||||
version='1.1b11.dev235',
|
||||
description='Distributed web crawling with browsers',
|
||||
url='https://github.com/internetarchive/brozzler',
|
||||
author='Noah Levitt',
|
||||
|
@ -146,6 +146,15 @@ def test_proxy_down():
|
||||
with pytest.raises(brozzler.ProxyError):
|
||||
worker._fetch_url(site, page)
|
||||
|
||||
# WARCPROX_WRITE_RECORD
|
||||
with pytest.raises(brozzler.ProxyError):
|
||||
worker._warcprox_write_record(
|
||||
warcprox_address=not_listening_proxy,
|
||||
url='test://proxy_down/warcprox_write_record',
|
||||
warc_type='metadata',
|
||||
content_type='text/plain',
|
||||
payload=b'''payload doesn't matter here''')
|
||||
|
||||
def test_start_stop_backwards_compat():
|
||||
site = brozzler.Site(None, {'seed': 'http://example.com/'})
|
||||
assert len(site.starts_and_stops) == 1
|
||||
|
Loading…
x
Reference in New Issue
Block a user