Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

use the new api with brozzler.thread_accept_exceptions()

Browse Source
This commit is contained in:
Noah Levitt 2017-04-24 20:02:34 -07:00
parent 0953e6972e
commit d916b68ab9
2 changed files with 18 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
brozzler/browser.py
View File

@ -427,8 +427,8 @@ class Browser:
self.websock_thread.on_request = on_request self.websock_thread.on_request = on_request
if on_response: if on_response:
self.websock_thread.on_response = on_response self.websock_thread.on_response = on_response
brozzler.thread_accept_exceptions()
try: try:
with brozzler.thread_accept_exceptions():
self.navigate_to_page( self.navigate_to_page(
page_url, extra_headers=extra_headers, page_url, extra_headers=extra_headers,
user_agent=user_agent, timeout=300) user_agent=user_agent, timeout=300)
@ -445,16 +445,13 @@ class Browser:
final_page_url = self.url() final_page_url = self.url()
return final_page_url, outlinks return final_page_url, outlinks
except brozzler.ReachedLimit: except brozzler.ReachedLimit:
brozzler.thread_block_exceptions()
# websock_thread has stashed the ReachedLimit exception with # websock_thread has stashed the ReachedLimit exception with
# more information, raise that one # more information, raise that one
raise self.websock_thread.reached_limit raise self.websock_thread.reached_limit
except websocket.WebSocketConnectionClosedException as e: except websocket.WebSocketConnectionClosedException as e:
brozzler.thread_block_exceptions()
self.logger.error('websocket closed, did chrome die?') self.logger.error('websocket closed, did chrome die?')
raise BrowsingException(e) raise BrowsingException(e)
finally: finally:
brozzler.thread_block_exceptions()
self.is_browsing = False self.is_browsing = False
self.websock_thread.on_request = None self.websock_thread.on_request = None
self.websock_thread.on_response = None self.websock_thread.on_response = None

3
brozzler/worker.py
View File

@ -234,9 +234,8 @@ class BrozzlerWorker:
def _try_youtube_dl(self, ydl, site, page): def _try_youtube_dl(self, ydl, site, page):
try: try:
self.logger.info("trying youtube-dl on {}".format(page)) self.logger.info("trying youtube-dl on {}".format(page))
brozzler.thread_accept_exceptions() with brozzler.thread_accept_exceptions():
info = ydl.extract_info(page.url) info = ydl.extract_info(page.url)
brozzler.thread_block_exceptions()
self._remember_videos(page, ydl.brozzler_spy) self._remember_videos(page, ydl.brozzler_spy)
# logging.info('XXX %s', json.dumps(info)) # logging.info('XXX %s', json.dumps(info))
if self._using_warcprox(site): if self._using_warcprox(site):