Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 07:45:50 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

wait 20 seconds to claim sites if none were avail-

Browse Source 
able last time, up from 0.5 seconds
this should lighten the load on rethinkdb considerably
This commit is contained in:
Noah Levitt 2018-08-31 15:23:59 -07:00
parent d0f5cd7168
commit 2d5c6681cf
1 changed files with 16 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
brozzler/worker.py
View File

@ -634,6 +634,7 @@ class BrozzlerWorker:
Raises:
NoBrowsersAvailable if none available
'''
# acquire_multi() raises NoBrowsersAvailable if none available
browsers = self._browser_pool.acquire_multi(
(self._browser_pool.num_available() + 1) // 2)
try:
@ -656,22 +657,26 @@ class BrozzlerWorker:
self._browser_pool.release(browsers[i])
def run(self):
self.logger.info("brozzler worker starting")
self.logger.notice("brozzler worker starting")
last_nothing_to_claim = 0
try:
while not self._shutdown.is_set():
self._service_heartbeat_if_due()
try:
self._start_browsing_some_sites()
except brozzler.browser.NoBrowsersAvailable:
logging.trace(
"all %s browsers are in use", self._max_browsers)
except brozzler.NothingToClaim:
logging.trace(
"all active sites are already claimed by a "
"brozzler worker")
if time.time() - last_nothing_to_claim > 20:
try:
self._start_browsing_some_sites()
except brozzler.browser.NoBrowsersAvailable:
logging.trace(
"all %s browsers are in use",
self._max_browsers)
except brozzler.NothingToClaim:
last_nothing_to_claim = time.time()
logging.trace(
"nothing to claim, all available active sites "
"are already claimed by a brozzler worker")
time.sleep(0.5)
self.logger.info("shutdown requested")
self.logger.notice("shutdown requested")
except r.ReqlError as e:
self.logger.error(
"caught rethinkdb exception, will try to proceed",