diff --git a/brozzler/frontier.py b/brozzler/frontier.py index 02c2dbb..1160f3c 100644 --- a/brozzler/frontier.py +++ b/brozzler/frontier.py @@ -197,12 +197,13 @@ class RethinkDbFrontier: else: raise brozzler.NothingToClaim - def enforce_time_limit(self, site): + def enforce_time_limit(self, site, session_time=0): ''' Raises `brozzler.ReachedTimeLimit` if appropriate. ''' - if (site.time_limit and site.time_limit > 0 - and (site.active_brozzling_time or 0) > site.time_limit): + if (site.time_limit + and site.time_limit > 0 + and (site.active_brozzling_time or 0) + session_time > site.time_limit): self.logger.debug( "site FINISHED_TIME_LIMIT! time_limit=%s " "active_brozzling_time=%s %s", site.time_limit, diff --git a/brozzler/worker.py b/brozzler/worker.py index 85feafc..872a3f3 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -515,7 +515,7 @@ class BrozzlerWorker: self._proxy_for(site), site) while time.time() - start < self.SITE_SESSION_MINUTES * 60: site.refresh() - self._frontier.enforce_time_limit(site) + self._frontier.enforce_time_limit(site, time.time() - start) self._frontier.honor_stop_request(site) page = self._frontier.claim_page(site, "%s:%s" % ( socket.gethostname(), browser.chrome.port))