Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

log exception when thread dies (seems to be dying silently sometimes)

Browse Source
This commit is contained in:
Noah Levitt 2015-09-03 01:04:41 +00:00
parent 839bf6f4ae
commit 6cda4739b8
1 changed files with 25 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
brozzler/worker.py
View File

@ -131,29 +131,32 @@ class BrozzlerWorker:
self._browser_pool.release(browser)
def run(self):
latest_state = None
while not self._shutdown_requested.is_set():
try:
browser = self._browser_pool.acquire()
try:
latest_state = None
while not self._shutdown_requested.is_set():
try:
site = self._frontier.claim_site()
self.logger.info("brozzling site %s", site)
ydl = self._youtube_dl(site)
th = threading.Thread(target=lambda: self._brozzle_site(browser, ydl, site),
name="BrowsingThread-{}".format(site.seed))
th.start()
except:
self._browser_pool.release(browser)
raise
except brozzler.browser.NoBrowsersAvailable:
if latest_state != "browsers-busy":
self.logger.info("all %s browsers are busy", self._max_browsers)
latest_state = "browsers-busy"
except brozzler.NothingToClaim:
if latest_state != "no-unclaimed-sites":
self.logger.info("no unclaimed sites to browse")
latest_state = "no-unclaimed-sites"
time.sleep(0.5)
browser = self._browser_pool.acquire()
try:
site = self._frontier.claim_site()
self.logger.info("brozzling site %s", site)
ydl = self._youtube_dl(site)
th = threading.Thread(target=lambda: self._brozzle_site(browser, ydl, site),
name="BrowsingThread-{}".format(site.seed))
th.start()
except:
self._browser_pool.release(browser)
raise
except brozzler.browser.NoBrowsersAvailable:
if latest_state != "browsers-busy":
self.logger.info("all %s browsers are busy", self._max_browsers)
latest_state = "browsers-busy"
except brozzler.NothingToClaim:
if latest_state != "no-unclaimed-sites":
self.logger.info("no unclaimed sites to browse")
latest_state = "no-unclaimed-sites"
time.sleep(0.5)
except:
self.logger.critical("thread exiting due to unexpected exception", exc_info=True)
def start(self):
th = threading.Thread(target=self.run, name="BrozzlerWorker")