Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-06 05:24:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Restructure browser.py to take screenshot after behavior script.

Browse source
This commit is contained in:
Adam Miller 2016-05-03 22:06:03 +00:00
parent f285be71fb
commit 61cec15fff
1 changed files with 21 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

26
brozzler/browser.py
View file

@ -174,6 +174,7 @@ class Browser:
self.on_screenshot = on_screenshot self.on_screenshot = on_screenshot
self.on_url_change = on_url_change self.on_url_change = on_url_change
self._waiting_on_scroll_to_top_msg_id = None
self._waiting_on_screenshot_msg_id = None self._waiting_on_screenshot_msg_id = None
self._waiting_on_document_url_msg_id = None self._waiting_on_document_url_msg_id = None
self._waiting_on_outlinks_msg_id = None self._waiting_on_outlinks_msg_id = None
@ -181,6 +182,7 @@ class Browser:
self._reached_limit = None self._reached_limit = None
self._aw_snap_hes_dead_jim = None self._aw_snap_hes_dead_jim = None
self._abort_browse_page = False self._abort_browse_page = False
self._has_screenshot = False
self._websock = websocket.WebSocketApp(self._websocket_url, self._websock = websocket.WebSocketApp(self._websocket_url,
on_open=self._visit_page, on_message=self._wrap_handle_message) on_open=self._visit_page, on_message=self._wrap_handle_message)
@ -226,6 +228,15 @@ class Browser:
raise BrowsingException("""chrome tab went "aw snap" or "he's dead jim"!""") raise BrowsingException("""chrome tab went "aw snap" or "he's dead jim"!""")
elif (self._behavior != None and self._behavior.is_finished() elif (self._behavior != None and self._behavior.is_finished()
or time.time() - self._start > Browser.HARD_TIMEOUT_SECONDS): or time.time() - self._start > Browser.HARD_TIMEOUT_SECONDS):
if ( not self._has_screenshot and not self._waiting_on_scroll_to_top_msg_id and not self._waiting_on_screenshot_msg_id):
self.logger.info("Behaviors finished, requesting screenshot %s", self.url)
self._waiting_on_scroll_to_top_msg_id = self.send_to_chrome(method="Runtime.evaluate",
params={"expression":"window.scrollTo(0, 0);"})
return False
elif not self._has_screenshot and (self._waiting_on_scroll_to_top_msg_id or self._waiting_on_screenshot_msg_id):
return False
if self._outlinks: if self._outlinks:
self.logger.info("got outlinks, finished browsing %s", self.url) self.logger.info("got outlinks, finished browsing %s", self.url)
return True return True
@ -246,6 +257,7 @@ class Browser:
return False return False
else: # self._waiting_on_outlinks_msg_id else: # self._waiting_on_outlinks_msg_id
return False return False
elif time.time() - self._start > Browser.HARD_TIMEOUT_SECONDS: elif time.time() - self._start > Browser.HARD_TIMEOUT_SECONDS:
return True return True
elif self._reached_limit: elif self._reached_limit:
@ -307,8 +319,10 @@ class Browser:
self.logger.info("reached limit %s", self._reached_limit) self.logger.info("reached limit %s", self._reached_limit)
def _page_load_event_fired(self, message): def _page_load_event_fired(self, message):
self.logger.info("Page.loadEventFired, requesting screenshot url={} message={}".format(self.url, message)) self.logger.info("Page.loadEventFired, moving on to starting behaviors url={}".format(self.url))
self._waiting_on_screenshot_msg_id = self.send_to_chrome(method="Page.captureScreenshot") self._behavior = Behavior(self.url, self)
self._behavior.start()
self._waiting_on_document_url_msg_id = self.send_to_chrome(method="Runtime.evaluate", params={"expression":"document.URL"}) self._waiting_on_document_url_msg_id = self.send_to_chrome(method="Runtime.evaluate", params={"expression":"document.URL"})
def _console_message_added(self, message): def _console_message_added(self, message):
@ -333,9 +347,11 @@ class Browser:
if self.on_screenshot: if self.on_screenshot:
self.on_screenshot(base64.b64decode(message["result"]["data"])) self.on_screenshot(base64.b64decode(message["result"]["data"]))
self._waiting_on_screenshot_msg_id = None self._waiting_on_screenshot_msg_id = None
self.logger.info("got screenshot, moving on to starting behaviors url={}".format(self.url)) self._has_screenshot = True
self._behavior = Behavior(self.url, self) self.logger.info("got screenshot, moving on to getting outlinks url={}".format(self.url))
self._behavior.start() elif message["id"] == self._waiting_on_scroll_to_top_msg_id:
self._waiting_on_screenshot_msg_id = self.send_to_chrome(method="Page.captureScreenshot")
self._waiting_on_scroll_to_top_msg_id = None
elif message["id"] == self._waiting_on_outlinks_msg_id: elif message["id"] == self._waiting_on_outlinks_msg_id:
self.logger.debug("got outlinks message=%s", message) self.logger.debug("got outlinks message=%s", message)
self._outlinks = frozenset(message["result"]["result"]["value"].split(" ")) self._outlinks = frozenset(message["result"]["result"]["value"].split(" "))