Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-02 19:46:36 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

bugfix

Browse source
This commit is contained in:
Vangelis Banos 2017-07-31 10:36:35 +00:00
parent 9c81a7bbda
commit 6259d03be1
1 changed files with 13 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

4
brozzler/worker.py
View file

@ -342,6 +342,8 @@ class BrozzlerWorker:
self.logger.error(
'youtube_dl raised exception on %s', page,
exc_info=True)
else:
ydl_spy = False
if self._needs_browsing(page, ydl_spy):
self.logger.info('needs browsing: %s', page)
@ -437,6 +439,7 @@ class BrozzlerWorker:
'proxy error fetching %s' % page.url) from e
def _needs_browsing(self, page, brozzler_spy):
if brozzler_spy:
final_bounces = brozzler_spy.final_bounces(page.url)
if not final_bounces:
return True
@ -447,6 +450,7 @@ class BrozzlerWorker:
return False
def _already_fetched(self, page, brozzler_spy):
if brozzler_spy:
for txn in brozzler_spy.final_bounces(page.url):
if (txn['method'] == 'GET' and txn['status_code'] == 200):
return True