Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

use the new api with brozzler.thread_accept_exceptions()

Browse Source
This commit is contained in:
Noah Levitt 2017-04-24 20:02:34 -07:00
parent 0953e6972e
commit d916b68ab9
2 changed files with 18 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
brozzler/browser.py
View File

@ -427,34 +427,31 @@ class Browser:
self.websock_thread.on_request = on_request self.websock_thread.on_request = on_request
if on_response: if on_response:
self.websock_thread.on_response = on_response self.websock_thread.on_response = on_response
brozzler.thread_accept_exceptions()
try: try:
self.navigate_to_page( with brozzler.thread_accept_exceptions():
page_url, extra_headers=extra_headers, self.navigate_to_page(
user_agent=user_agent, timeout=300) page_url, extra_headers=extra_headers,
if password: user_agent=user_agent, timeout=300)
self.try_login(username, password, timeout=300) if password:
if on_screenshot: self.try_login(username, password, timeout=300)
jpeg_bytes = self.screenshot() if on_screenshot:
on_screenshot(jpeg_bytes) jpeg_bytes = self.screenshot()
behavior_script = brozzler.behavior_script( on_screenshot(jpeg_bytes)
page_url, behavior_parameters) behavior_script = brozzler.behavior_script(
self.run_behavior(behavior_script, timeout=900) page_url, behavior_parameters)
outlinks = self.extract_outlinks() self.run_behavior(behavior_script, timeout=900)
self.visit_hashtags(page_url, hashtags, outlinks) outlinks = self.extract_outlinks()
final_page_url = self.url() self.visit_hashtags(page_url, hashtags, outlinks)
return final_page_url, outlinks final_page_url = self.url()
return final_page_url, outlinks
except brozzler.ReachedLimit: except brozzler.ReachedLimit:
brozzler.thread_block_exceptions()
# websock_thread has stashed the ReachedLimit exception with # websock_thread has stashed the ReachedLimit exception with
# more information, raise that one # more information, raise that one
raise self.websock_thread.reached_limit raise self.websock_thread.reached_limit
except websocket.WebSocketConnectionClosedException as e: except websocket.WebSocketConnectionClosedException as e:
brozzler.thread_block_exceptions()
self.logger.error('websocket closed, did chrome die?') self.logger.error('websocket closed, did chrome die?')
raise BrowsingException(e) raise BrowsingException(e)
finally: finally:
brozzler.thread_block_exceptions()
self.is_browsing = False self.is_browsing = False
self.websock_thread.on_request = None self.websock_thread.on_request = None
self.websock_thread.on_response = None self.websock_thread.on_response = None

5
brozzler/worker.py
View File

@ -234,9 +234,8 @@ class BrozzlerWorker:
def _try_youtube_dl(self, ydl, site, page): def _try_youtube_dl(self, ydl, site, page):
try: try:
self.logger.info("trying youtube-dl on {}".format(page)) self.logger.info("trying youtube-dl on {}".format(page))
brozzler.thread_accept_exceptions() with brozzler.thread_accept_exceptions():
info = ydl.extract_info(page.url) info = ydl.extract_info(page.url)
brozzler.thread_block_exceptions()
self._remember_videos(page, ydl.brozzler_spy) self._remember_videos(page, ydl.brozzler_spy)
# logging.info('XXX %s', json.dumps(info)) # logging.info('XXX %s', json.dumps(info))
if self._using_warcprox(site): if self._using_warcprox(site):