Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-02 03:26:11 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

change time limit enforcement

Browse source 
enforce time limit based on all the time that a site was in active
rotation, including time it spent waiting for its turn to be brozzled;
this undoes the change from b9640b8a30, because now it seems that
was the wrong decision (brozzler jobs with many seeds and low
max_claimed_sites hanging around forever)
This commit is contained in:
Noah Levitt 2018-11-12 16:21:38 -08:00
parent 15610fa990
commit 05fab8b909
3 changed files with 11 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

24
tests/test_frontier.py
View file

@ -21,6 +21,7 @@ limitations under the License.
import argparse
import datetime
import logging
import time
import doublethink
import pytest
@ -375,15 +376,10 @@ def test_time_limit():
assert site.starts_and_stops[1]['start']
assert site.starts_and_stops[1]['stop'] is None
# time limit not reached yet
# no time limit set
frontier.enforce_time_limit(site)
assert site.status == 'ACTIVE'
assert len(site.starts_and_stops) == 2
assert site.starts_and_stops[1]['start']
assert site.starts_and_stops[1]['stop'] is None
site.time_limit = 0.1
site.time_limit = 10
site.claimed = True
site.save()
@ -394,19 +390,11 @@ def test_time_limit():
assert site.starts_and_stops[1]['start']
assert site.starts_and_stops[1]['stop'] is None
site.active_brozzling_time = 0.2 # this is why the time limit will be hit
site.time_limit = 0.1
time.sleep(0.1)
try:
with pytest.raises(brozzler.ReachedTimeLimit):
frontier.enforce_time_limit(site)
except brozzler.ReachedTimeLimit:
frontier.finished(site, 'FINISHED_TIME_LIMIT')
assert site.status == 'FINISHED_TIME_LIMIT'
assert not site.claimed
assert len(site.starts_and_stops) == 2
assert site.starts_and_stops[1]['start']
assert site.starts_and_stops[1]['stop']
assert site.starts_and_stops[1]['stop'] > site.starts_and_stops[0]['start']
def test_field_defaults():
rr = doublethink.Rethinker('localhost', db='ignoreme')