Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-20 12:54:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

make sure site is not "claimed" when it's finished

Browse source
This commit is contained in:
Noah Levitt 2017-02-03 16:40:15 -08:00
parent a60878c5a7
commit 14e312e4c4
3 changed files with 5 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
brozzler/frontier.py
View file

@ -285,6 +285,8 @@ class RethinkDbFrontier:
def finished(self, site, status): def finished(self, site, status):
self.logger.info("%s %s", status, site) self.logger.info("%s %s", status, site)
site.status = status site.status = status
site.claimed = False
site.last_disclaimed = rethinkstuff.utcnow()
site.starts_and_stops[-1]["stop"] = rethinkstuff.utcnow() site.starts_and_stops[-1]["stop"] = rethinkstuff.utcnow()
self.update_site(site) self.update_site(site)
if site.job_id: if site.job_id:

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b9.dev183', version='1.1b9.dev184',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',

2
tests/test_frontier.py
View file

@ -160,12 +160,14 @@ def test_time_limit():
assert site.starts_and_stops[1]['stop'] is None assert site.starts_and_stops[1]['stop'] is None
site.time_limit = 0.1 site.time_limit = 0.1
site.claimed = True
frontier.update_site(site) frontier.update_site(site)
time.sleep(0.1) time.sleep(0.1)
frontier._enforce_time_limit(site) frontier._enforce_time_limit(site)
assert site.status == 'FINISHED_TIME_LIMIT' assert site.status == 'FINISHED_TIME_LIMIT'
assert not site.claimed
assert len(site.starts_and_stops) == 2 assert len(site.starts_and_stops) == 2
assert site.starts_and_stops[1]['start'] assert site.starts_and_stops[1]['start']
assert site.starts_and_stops[1]['stop'] assert site.starts_and_stops[1]['stop']