mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-06-20 12:54:23 -04:00
have _warcprox_write_record also raise ProxyError when appropriate, and test this
This commit is contained in:
parent
ac972d399f
commit
5603ff5380
3 changed files with 12 additions and 1 deletions
|
@ -205,6 +205,8 @@ class BrozzlerWorker:
|
||||||
'got "%s %s" response on warcprox '
|
'got "%s %s" response on warcprox '
|
||||||
'WARCPROX_WRITE_RECORD request (expected 204)',
|
'WARCPROX_WRITE_RECORD request (expected 204)',
|
||||||
e.getcode(), e.info())
|
e.getcode(), e.info())
|
||||||
|
except urllib.error.URLError as e:
|
||||||
|
raise brozzler.ProxyError('_warcprox_write_record: %s', e)
|
||||||
|
|
||||||
def _remember_videos(self, page, ydl_spy):
|
def _remember_videos(self, page, ydl_spy):
|
||||||
if not 'videos' in page:
|
if not 'videos' in page:
|
||||||
|
|
2
setup.py
2
setup.py
|
@ -32,7 +32,7 @@ def find_package_data(package):
|
||||||
|
|
||||||
setuptools.setup(
|
setuptools.setup(
|
||||||
name='brozzler',
|
name='brozzler',
|
||||||
version='1.1b11.dev234',
|
version='1.1b11.dev235',
|
||||||
description='Distributed web crawling with browsers',
|
description='Distributed web crawling with browsers',
|
||||||
url='https://github.com/internetarchive/brozzler',
|
url='https://github.com/internetarchive/brozzler',
|
||||||
author='Noah Levitt',
|
author='Noah Levitt',
|
||||||
|
|
|
@ -146,6 +146,15 @@ def test_proxy_down():
|
||||||
with pytest.raises(brozzler.ProxyError):
|
with pytest.raises(brozzler.ProxyError):
|
||||||
worker._fetch_url(site, page)
|
worker._fetch_url(site, page)
|
||||||
|
|
||||||
|
# WARCPROX_WRITE_RECORD
|
||||||
|
with pytest.raises(brozzler.ProxyError):
|
||||||
|
worker._warcprox_write_record(
|
||||||
|
warcprox_address=not_listening_proxy,
|
||||||
|
url='test://proxy_down/warcprox_write_record',
|
||||||
|
warc_type='metadata',
|
||||||
|
content_type='text/plain',
|
||||||
|
payload=b'''payload doesn't matter here''')
|
||||||
|
|
||||||
def test_start_stop_backwards_compat():
|
def test_start_stop_backwards_compat():
|
||||||
site = brozzler.Site(None, {'seed': 'http://example.com/'})
|
site = brozzler.Site(None, {'seed': 'http://example.com/'})
|
||||||
assert len(site.starts_and_stops) == 1
|
assert len(site.starts_and_stops) == 1
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue