Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-12-17 01:24:06 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

wait 20 seconds to claim sites if none were avail-

Browse source 
able last time, up from 0.5 seconds
this should lighten the load on rethinkdb considerably
This commit is contained in:
Noah Levitt 2018-08-31 15:23:59 -07:00
parent d0f5cd7168
commit 2d5c6681cf
1 changed files with 16 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

27
brozzler/worker.py
View file

@ -634,6 +634,7 @@ class BrozzlerWorker:
Raises: Raises:
NoBrowsersAvailable if none available NoBrowsersAvailable if none available
''' '''
# acquire_multi() raises NoBrowsersAvailable if none available
browsers = self._browser_pool.acquire_multi( browsers = self._browser_pool.acquire_multi(
(self._browser_pool.num_available() + 1) // 2) (self._browser_pool.num_available() + 1) // 2)
try: try:
@ -656,22 +657,26 @@ class BrozzlerWorker:
self._browser_pool.release(browsers[i]) self._browser_pool.release(browsers[i])
def run(self): def run(self):
self.logger.info("brozzler worker starting") self.logger.notice("brozzler worker starting")
last_nothing_to_claim = 0
try: try:
while not self._shutdown.is_set(): while not self._shutdown.is_set():
self._service_heartbeat_if_due() self._service_heartbeat_if_due()
try: if time.time() - last_nothing_to_claim > 20:
self._start_browsing_some_sites() try:
except brozzler.browser.NoBrowsersAvailable: self._start_browsing_some_sites()
logging.trace( except brozzler.browser.NoBrowsersAvailable:
"all %s browsers are in use", self._max_browsers) logging.trace(
except brozzler.NothingToClaim: "all %s browsers are in use",
logging.trace( self._max_browsers)
"all active sites are already claimed by a " except brozzler.NothingToClaim:
"brozzler worker") last_nothing_to_claim = time.time()
logging.trace(
"nothing to claim, all available active sites "
"are already claimed by a brozzler worker")
time.sleep(0.5) time.sleep(0.5)
self.logger.info("shutdown requested") self.logger.notice("shutdown requested")
except r.ReqlError as e: except r.ReqlError as e:
self.logger.error( self.logger.error(
"caught rethinkdb exception, will try to proceed", "caught rethinkdb exception, will try to proceed",