Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

log exception when thread dies (seems to be dying silently sometimes)

Browse Source
This commit is contained in:
Noah Levitt 2015-09-03 01:04:41 +00:00
parent 839bf6f4ae
commit 6cda4739b8
1 changed files with 25 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
brozzler/worker.py
View File

@ -131,29 +131,32 @@ class BrozzlerWorker:
self._browser_pool.release(browser) self._browser_pool.release(browser)
def run(self): def run(self):
latest_state = None try:
while not self._shutdown_requested.is_set(): latest_state = None
try: while not self._shutdown_requested.is_set():
browser = self._browser_pool.acquire()
try: try:
site = self._frontier.claim_site() browser = self._browser_pool.acquire()
self.logger.info("brozzling site %s", site) try:
ydl = self._youtube_dl(site) site = self._frontier.claim_site()
th = threading.Thread(target=lambda: self._brozzle_site(browser, ydl, site), self.logger.info("brozzling site %s", site)
name="BrowsingThread-{}".format(site.seed)) ydl = self._youtube_dl(site)
th.start() th = threading.Thread(target=lambda: self._brozzle_site(browser, ydl, site),
except: name="BrowsingThread-{}".format(site.seed))
self._browser_pool.release(browser) th.start()
raise except:
except brozzler.browser.NoBrowsersAvailable: self._browser_pool.release(browser)
if latest_state != "browsers-busy": raise
self.logger.info("all %s browsers are busy", self._max_browsers) except brozzler.browser.NoBrowsersAvailable:
latest_state = "browsers-busy" if latest_state != "browsers-busy":
except brozzler.NothingToClaim: self.logger.info("all %s browsers are busy", self._max_browsers)
if latest_state != "no-unclaimed-sites": latest_state = "browsers-busy"
self.logger.info("no unclaimed sites to browse") except brozzler.NothingToClaim:
latest_state = "no-unclaimed-sites" if latest_state != "no-unclaimed-sites":
time.sleep(0.5) self.logger.info("no unclaimed sites to browse")
latest_state = "no-unclaimed-sites"
time.sleep(0.5)
except:
self.logger.critical("thread exiting due to unexpected exception", exc_info=True)
def start(self): def start(self):
th = threading.Thread(target=self.run, name="BrozzlerWorker") th = threading.Thread(target=self.run, name="BrozzlerWorker")