Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-21 13:24:20 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #136 from nlevitt/revert-time-limit

Browse source 
change time limit enforcement
This commit is contained in:
jkafader 2018-11-29 12:29:35 -08:00 • committed by GitHub
commit 9c27e829aa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 11 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

10
brozzler/frontier.py
View file

@ -152,17 +152,15 @@ class RethinkDbFrontier:
else: else:
raise brozzler.NothingToClaim raise brozzler.NothingToClaim
def enforce_time_limit(self, site, session_time=0): def enforce_time_limit(self, site):
''' '''
Raises `brozzler.ReachedTimeLimit` if appropriate. Raises `brozzler.ReachedTimeLimit` if appropriate.
''' '''
if (site.time_limit if (site.time_limit and site.time_limit > 0
and site.time_limit > 0 and site.elapsed() > site.time_limit):
and (site.active_brozzling_time or 0) + session_time > site.time_limit):
self.logger.debug( self.logger.debug(
"site FINISHED_TIME_LIMIT! time_limit=%s " "site FINISHED_TIME_LIMIT! time_limit=%s "
"active_brozzling_time=%s %s", site.time_limit, "elapsed=%s %s", site.time_limit, site.elapsed(), site)
site.active_brozzling_time, site)
raise brozzler.ReachedTimeLimit raise brozzler.ReachedTimeLimit
def claim_page(self, site, worker_id): def claim_page(self, site, worker_id):

2
brozzler/worker.py
View file

@ -348,7 +348,7 @@ class BrozzlerWorker:
self._proxy_for(site), site) self._proxy_for(site), site)
while time.time() - start < self.SITE_SESSION_MINUTES * 60: while time.time() - start < self.SITE_SESSION_MINUTES * 60:
site.refresh() site.refresh()
self._frontier.enforce_time_limit(site, time.time() - start) self._frontier.enforce_time_limit(site)
self._frontier.honor_stop_request(site) self._frontier.honor_stop_request(site)
page = self._frontier.claim_page(site, "%s:%s" % ( page = self._frontier.claim_page(site, "%s:%s" % (
socket.gethostname(), browser.chrome.port)) socket.gethostname(), browser.chrome.port))

24
tests/test_frontier.py
View file

@ -21,6 +21,7 @@ limitations under the License.
import argparse import argparse
import datetime import datetime
import logging import logging
import time
import doublethink import doublethink
import pytest import pytest
@ -375,15 +376,10 @@ def test_time_limit():
assert site.starts_and_stops[1]['start'] assert site.starts_and_stops[1]['start']
assert site.starts_and_stops[1]['stop'] is None assert site.starts_and_stops[1]['stop'] is None
# time limit not reached yet # no time limit set
frontier.enforce_time_limit(site) frontier.enforce_time_limit(site)
assert site.status == 'ACTIVE' site.time_limit = 10
assert len(site.starts_and_stops) == 2
assert site.starts_and_stops[1]['start']
assert site.starts_and_stops[1]['stop'] is None
site.time_limit = 0.1
site.claimed = True site.claimed = True
site.save() site.save()
@ -394,19 +390,11 @@ def test_time_limit():
assert site.starts_and_stops[1]['start'] assert site.starts_and_stops[1]['start']
assert site.starts_and_stops[1]['stop'] is None assert site.starts_and_stops[1]['stop'] is None
site.active_brozzling_time = 0.2 # this is why the time limit will be hit site.time_limit = 0.1
time.sleep(0.1)
try: with pytest.raises(brozzler.ReachedTimeLimit):
frontier.enforce_time_limit(site) frontier.enforce_time_limit(site)
except brozzler.ReachedTimeLimit:
frontier.finished(site, 'FINISHED_TIME_LIMIT')
assert site.status == 'FINISHED_TIME_LIMIT'
assert not site.claimed
assert len(site.starts_and_stops) == 2
assert site.starts_and_stops[1]['start']
assert site.starts_and_stops[1]['stop']
assert site.starts_and_stops[1]['stop'] > site.starts_and_stops[0]['start']
def test_field_defaults(): def test_field_defaults():
rr = doublethink.Rethinker('localhost', db='ignoreme') rr = doublethink.Rethinker('localhost', db='ignoreme')