diff --git a/brozzler/frontier.py b/brozzler/frontier.py index 856b8bd..0ae91eb 100644 --- a/brozzler/frontier.py +++ b/brozzler/frontier.py @@ -285,6 +285,8 @@ class RethinkDbFrontier: def finished(self, site, status): self.logger.info("%s %s", status, site) site.status = status + site.claimed = False + site.last_disclaimed = rethinkstuff.utcnow() site.starts_and_stops[-1]["stop"] = rethinkstuff.utcnow() self.update_site(site) if site.job_id: diff --git a/setup.py b/setup.py index 4a3821a..bd6a6fc 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.1b9.dev183', + version='1.1b9.dev184', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt', diff --git a/tests/test_frontier.py b/tests/test_frontier.py index 4553835..3b95fc6 100644 --- a/tests/test_frontier.py +++ b/tests/test_frontier.py @@ -160,12 +160,14 @@ def test_time_limit(): assert site.starts_and_stops[1]['stop'] is None site.time_limit = 0.1 + site.claimed = True frontier.update_site(site) time.sleep(0.1) frontier._enforce_time_limit(site) assert site.status == 'FINISHED_TIME_LIMIT' + assert not site.claimed assert len(site.starts_and_stops) == 2 assert site.starts_and_stops[1]['start'] assert site.starts_and_stops[1]['stop']