Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 00:29:53 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

use the new api with brozzler.thread_accept_exceptions()

Browse Source
This commit is contained in:
Noah Levitt 2017-04-24 20:02:34 -07:00
parent 0953e6972e
commit d916b68ab9
2 changed files with 18 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
brozzler/browser.py
View File

@ -427,34 +427,31 @@ class Browser:
self.websock_thread.on_request = on_request
if on_response:
self.websock_thread.on_response = on_response
brozzler.thread_accept_exceptions()
try:
self.navigate_to_page(
page_url, extra_headers=extra_headers,
user_agent=user_agent, timeout=300)
if password:
self.try_login(username, password, timeout=300)
if on_screenshot:
jpeg_bytes = self.screenshot()
on_screenshot(jpeg_bytes)
behavior_script = brozzler.behavior_script(
page_url, behavior_parameters)
self.run_behavior(behavior_script, timeout=900)
outlinks = self.extract_outlinks()
self.visit_hashtags(page_url, hashtags, outlinks)
final_page_url = self.url()
return final_page_url, outlinks
with brozzler.thread_accept_exceptions():
self.navigate_to_page(
page_url, extra_headers=extra_headers,
user_agent=user_agent, timeout=300)
if password:
self.try_login(username, password, timeout=300)
if on_screenshot:
jpeg_bytes = self.screenshot()
on_screenshot(jpeg_bytes)
behavior_script = brozzler.behavior_script(
page_url, behavior_parameters)
self.run_behavior(behavior_script, timeout=900)
outlinks = self.extract_outlinks()
self.visit_hashtags(page_url, hashtags, outlinks)
final_page_url = self.url()
return final_page_url, outlinks
except brozzler.ReachedLimit:
brozzler.thread_block_exceptions()
# websock_thread has stashed the ReachedLimit exception with
# more information, raise that one
raise self.websock_thread.reached_limit
except websocket.WebSocketConnectionClosedException as e:
brozzler.thread_block_exceptions()
self.logger.error('websocket closed, did chrome die?')
raise BrowsingException(e)
finally:
brozzler.thread_block_exceptions()
self.is_browsing = False
self.websock_thread.on_request = None
self.websock_thread.on_response = None

5
brozzler/worker.py
View File

@ -234,9 +234,8 @@ class BrozzlerWorker:
def _try_youtube_dl(self, ydl, site, page):
try:
self.logger.info("trying youtube-dl on {}".format(page))
brozzler.thread_accept_exceptions()
info = ydl.extract_info(page.url)
brozzler.thread_block_exceptions()
with brozzler.thread_accept_exceptions():
info = ydl.extract_info(page.url)
self._remember_videos(page, ydl.brozzler_spy)
# logging.info('XXX %s', json.dumps(info))
if self._using_warcprox(site):