Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

apparently response.status doesn't work sometimes; response.getcode() is documented so hopefully it keeps working

Browse Source
This commit is contained in:
Noah Levitt 2016-10-25 17:50:49 -07:00
parent 70ce642bee
commit 332912acd7
2 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
brozzler/worker.py
View File

@ -53,7 +53,7 @@ class YoutubeDLSpy(urllib.request.BaseHandler):
self.reset() self.reset()
def _http_response(self, request, response): def _http_response(self, request, response):
self.transactions.append(YoutubeDLSpy.Transaction(request,response)) self.transactions.append(YoutubeDLSpy.Transaction(request, response))
return response return response
http_response = https_response = _http_response http_response = https_response = _http_response
@ -175,11 +175,11 @@ class BrozzlerWorker:
try: try:
with urllib.request.urlopen(request) as response: with urllib.request.urlopen(request) as response:
if response.status != 204: if response.getcode() != 204:
self.logger.warn( self.logger.warn(
'got "%s %s" response on warcprox ' 'got "%s %s" response on warcprox '
'WARCPROX_WRITE_RECORD request (expected 204)', 'WARCPROX_WRITE_RECORD request (expected 204)',
response.status, response.reason) response.getcode(), response.reason)
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
self.logger.warn( self.logger.warn(
'got "%s %s" response on warcprox ' 'got "%s %s" response on warcprox '
@ -311,7 +311,7 @@ class BrozzlerWorker:
def _already_fetched(self, page, brozzler_spy): def _already_fetched(self, page, brozzler_spy):
for txn in brozzler_spy.final_bounces(page.url): for txn in brozzler_spy.final_bounces(page.url):
if (txn.request.get_method() == 'GET' if (txn.request.get_method() == 'GET'
and txn.response.status == 200): and txn.response.getcode() == 200):
return True return True
return False return False

2
setup.py
View File

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b7.dev106', version='1.1b7.dev107',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',