Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-20 12:54:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

set a timeout on warcprox_write_record request

Browse source
This commit is contained in:
Noah Levitt 2017-12-27 15:52:55 -08:00
parent cc6297ef60
commit 503771d653
2 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
brozzler/worker.py
View file

@ -225,7 +225,7 @@ class BrozzlerWorker:
request.set_proxy(warcprox_address, "http") request.set_proxy(warcprox_address, "http")
try: try:
with urllib.request.urlopen(request) as response: with urllib.request.urlopen(request, timeout=600) as response:
if response.getcode() != 204: if response.getcode() != 204:
self.logger.warn( self.logger.warn(
'got "%s %s" response on warcprox ' 'got "%s %s" response on warcprox '

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b12.dev275', version='1.1b12.dev276',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',