Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 23:56:34 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix timely time limit enforcement

Browse Source 
by including current brozzling session duration in time accounting
This commit is contained in:
Noah Levitt 2018-03-05 17:05:41 -08:00
parent 318ae13bcb
commit 4daac3dfc5
2 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
brozzler/frontier.py
View File

@ -197,12 +197,13 @@ class RethinkDbFrontier:
else:
raise brozzler.NothingToClaim
def enforce_time_limit(self, site):
def enforce_time_limit(self, site, session_time=0):
'''
Raises `brozzler.ReachedTimeLimit` if appropriate.
'''
if (site.time_limit and site.time_limit > 0
and (site.active_brozzling_time or 0) > site.time_limit):
if (site.time_limit
and site.time_limit > 0
and (site.active_brozzling_time or 0) + session_time > site.time_limit):
self.logger.debug(
"site FINISHED_TIME_LIMIT! time_limit=%s "
"active_brozzling_time=%s %s", site.time_limit,

2
brozzler/worker.py
View File

@ -515,7 +515,7 @@ class BrozzlerWorker:
self._proxy_for(site), site)
while time.time() - start < self.SITE_SESSION_MINUTES * 60:
site.refresh()
self._frontier.enforce_time_limit(site)
self._frontier.enforce_time_limit(site, time.time() - start)
self._frontier.honor_stop_request(site)
page = self._frontier.claim_page(site, "%s:%s" % (
socket.gethostname(), browser.chrome.port))