Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-21 08:06:27 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

except log and return []

Browse Source
This commit is contained in:
Barbara Miller 2018-09-24 17:33:38 -07:00
parent 156ec0caa1
commit d9f7997a40
1 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
brozzler/worker.py
View File

@ -209,9 +209,13 @@ class BrozzlerWorker:
if self._needs_browsing(page, ydl_fetches):
self.logger.info('needs browsing: %s', page)
outlinks = self._browse_page(browser, site, page, on_screenshot,
on_request)
return outlinks
try:
outlinks = self._browse_page(browser, site, page, on_screenshot,
on_request)
return outlinks
except brozzler.PageInterstitialShown:
self.logger.info('page interstitial shown (http auth): %s', page)
return []
else:
if not self._already_fetched(page, ydl_fetches):
self.logger.info('needs fetch: %s', page)
@ -375,8 +379,6 @@ class BrozzlerWorker:
self._frontier.finished(site, "FINISHED_TIME_LIMIT")
except brozzler.CrawlStopped:
self._frontier.finished(site, "FINISHED_STOP_REQUESTED")
except brozzler.PageInterstitialShown:
self.logger.info("{} shut down after unsupported http auth request".format(browser))
# except brozzler.browser.BrowsingAborted:
# self.logger.info("{} shut down".format(browser))
except brozzler.ProxyError as e: