Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-05-02 06:36:20 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

disable the re-claiming of sites that are marked claimed from more than an hour ago, because sometimes pages legitimately take longer than an hour to brozzle; working on a better solution to this issue

Browse source
This commit is contained in:
Noah Levitt 2017-06-19 11:21:02 -07:00
parent 7ae22381ef
commit 6bae53e646
3 changed files with 19 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

11
tests/test_frontier.py
View file

@ -723,14 +723,17 @@ def test_claim_site():
with pytest.raises(brozzler.NothingToClaim):
claimed_site = frontier.claim_site(worker_id='test_claim_site')
# site last_claimed more than 1 hour ago can be reclaimed
### temporarily changing this behavior
### # site last_claimed more than 1 hour ago can be reclaimed
site = claimed_site
claimed_site = None
site.last_claimed = doublethink.utcnow() - datetime.timedelta(minutes=65)
site.save()
claimed_site = frontier.claim_site(worker_id='test_claim_site')
assert claimed_site.id == site.id
### claimed_site = frontier.claim_site(worker_id='test_claim_site')
### assert claimed_site.id == site.id
with pytest.raises(brozzler.NothingToClaim):
claimed_site = frontier.claim_site(worker_id='test_claim_site')
# clean up
rr.table('sites').get(claimed_site.id).delete().run()
rr.table('sites').get(site.id).delete().run()