Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 23:56:34 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

limit the number of times we try to navigate_to_page, then let it go.

Browse Source
This commit is contained in:
Daniel Bicho 2019-11-28 16:20:21 +00:00
parent 060adaffd0
commit 746937f19c
1 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
brozzler/browser.py
View File

@ -573,12 +573,16 @@ class Browser:
params={'userAgent': user_agent})
def navigate_to_page(self, page_url, timeout=300):
self.logger.info('navigating to page %s', page_url)
self.websock_thread.got_page_load_event = None
self.send_to_chrome(method='Page.navigate', params={'url': page_url})
self._wait_for(
lambda: self.websock_thread.got_page_load_event,
timeout=timeout)
try:
for i in range(2):
self.logger.info('navigating to page %s', page_url)
self.websock_thread.got_page_load_event = None
self.send_to_chrome(method='Page.navigate', params={'url': page_url})
self._wait_for(
lambda: self.websock_thread.got_page_load_event,
timeout=timeout)
except BrowsingTimeout as e:
logging.error('attempt %s/2: %s', i + 1, e)
def extract_outlinks(self, timeout=60):
self.logger.info('extracting outlinks')