diff --git a/brozzler/worker.py b/brozzler/worker.py index c999814..7e7d974 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -209,9 +209,13 @@ class BrozzlerWorker: if self._needs_browsing(page, ydl_fetches): self.logger.info('needs browsing: %s', page) - outlinks = self._browse_page(browser, site, page, on_screenshot, - on_request) - return outlinks + try: + outlinks = self._browse_page(browser, site, page, on_screenshot, + on_request) + return outlinks + except brozzler.PageInterstitialShown: + self.logger.info('page interstitial shown (http auth): %s', page) + return [] else: if not self._already_fetched(page, ydl_fetches): self.logger.info('needs fetch: %s', page) @@ -375,8 +379,6 @@ class BrozzlerWorker: self._frontier.finished(site, "FINISHED_TIME_LIMIT") except brozzler.CrawlStopped: self._frontier.finished(site, "FINISHED_STOP_REQUESTED") - except brozzler.PageInterstitialShown: - self.logger.info("{} shut down after unsupported http auth request".format(browser)) # except brozzler.browser.BrowsingAborted: # self.logger.info("{} shut down".format(browser)) except brozzler.ProxyError as e: