Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-19 15:25:59 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

make sure site is not "claimed" when it's finished

Browse Source
This commit is contained in:
Noah Levitt 2017-02-03 16:40:15 -08:00
parent a60878c5a7
commit 14e312e4c4
3 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
brozzler/frontier.py
View File

@ -285,6 +285,8 @@ class RethinkDbFrontier:
def finished(self, site, status):
self.logger.info("%s %s", status, site)
site.status = status
site.claimed = False
site.last_disclaimed = rethinkstuff.utcnow()
site.starts_and_stops[-1]["stop"] = rethinkstuff.utcnow()
self.update_site(site)
if site.job_id:

2
setup.py
View File

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup(
name='brozzler',
version='1.1b9.dev183',
version='1.1b9.dev184',
description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler',
author='Noah Levitt',

2
tests/test_frontier.py
View File

@ -160,12 +160,14 @@ def test_time_limit():
assert site.starts_and_stops[1]['stop'] is None
site.time_limit = 0.1
site.claimed = True
frontier.update_site(site)
time.sleep(0.1)
frontier._enforce_time_limit(site)
assert site.status == 'FINISHED_TIME_LIMIT'
assert not site.claimed
assert len(site.starts_and_stops) == 2
assert site.starts_and_stops[1]['start']
assert site.starts_and_stops[1]['stop']