diff --git a/brozzler/frontier.py b/brozzler/frontier.py index 5272a88..3826abf 100644 --- a/brozzler/frontier.py +++ b/brozzler/frontier.py @@ -152,17 +152,15 @@ class RethinkDbFrontier: else: raise brozzler.NothingToClaim - def enforce_time_limit(self, site, session_time=0): + def enforce_time_limit(self, site): ''' Raises `brozzler.ReachedTimeLimit` if appropriate. ''' - if (site.time_limit - and site.time_limit > 0 - and (site.active_brozzling_time or 0) + session_time > site.time_limit): + if (site.time_limit and site.time_limit > 0 + and site.elapsed() > site.time_limit): self.logger.debug( "site FINISHED_TIME_LIMIT! time_limit=%s " - "active_brozzling_time=%s %s", site.time_limit, - site.active_brozzling_time, site) + "elapsed=%s %s", site.time_limit, site.elapsed(), site) raise brozzler.ReachedTimeLimit def claim_page(self, site, worker_id): diff --git a/brozzler/worker.py b/brozzler/worker.py index 4aad2a3..8121770 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -345,7 +345,7 @@ class BrozzlerWorker: self._proxy_for(site), site) while time.time() - start < self.SITE_SESSION_MINUTES * 60: site.refresh() - self._frontier.enforce_time_limit(site, time.time() - start) + self._frontier.enforce_time_limit(site) self._frontier.honor_stop_request(site) page = self._frontier.claim_page(site, "%s:%s" % ( socket.gethostname(), browser.chrome.port)) diff --git a/tests/test_frontier.py b/tests/test_frontier.py index d66773e..800da1e 100644 --- a/tests/test_frontier.py +++ b/tests/test_frontier.py @@ -21,6 +21,7 @@ limitations under the License. import argparse import datetime import logging +import time import doublethink import pytest @@ -375,15 +376,10 @@ def test_time_limit(): assert site.starts_and_stops[1]['start'] assert site.starts_and_stops[1]['stop'] is None - # time limit not reached yet + # no time limit set frontier.enforce_time_limit(site) - assert site.status == 'ACTIVE' - assert len(site.starts_and_stops) == 2 - assert site.starts_and_stops[1]['start'] - assert site.starts_and_stops[1]['stop'] is None - - site.time_limit = 0.1 + site.time_limit = 10 site.claimed = True site.save() @@ -394,19 +390,11 @@ def test_time_limit(): assert site.starts_and_stops[1]['start'] assert site.starts_and_stops[1]['stop'] is None - site.active_brozzling_time = 0.2 # this is why the time limit will be hit + site.time_limit = 0.1 + time.sleep(0.1) - try: + with pytest.raises(brozzler.ReachedTimeLimit): frontier.enforce_time_limit(site) - except brozzler.ReachedTimeLimit: - frontier.finished(site, 'FINISHED_TIME_LIMIT') - - assert site.status == 'FINISHED_TIME_LIMIT' - assert not site.claimed - assert len(site.starts_and_stops) == 2 - assert site.starts_and_stops[1]['start'] - assert site.starts_and_stops[1]['stop'] - assert site.starts_and_stops[1]['stop'] > site.starts_and_stops[0]['start'] def test_field_defaults(): rr = doublethink.Rethinker('localhost', db='ignoreme')