Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-05-15 04:52:22 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix case where rethinkdb page already has claimed=True

Browse source
This commit is contained in:
Noah Levitt 2016-06-29 19:29:18 -05:00
parent ffcf26b6c9
commit 2e687b65fb
2 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

10
brozzler/frontier.py
View file

@ -197,12 +197,12 @@ class RethinkDbFrontier:
1).update({
"claimed":True,
"last_claimed_by":worker_id},
return_changes=True).run()
self._vet_result(result, replaced=[0,1])
if result["replaced"] == 1:
return brozzler.Page(**result["changes"][0]["new_val"])
else:
return_changes="always").run()
self._vet_result(result, unchanged=[0,1], replaced=[0,1])
if result["unchanged"] == 0 and result["replaced"] == 0:
raise brozzler.NothingToClaim
else:
return brozzler.Page(**result["changes"][0]["new_val"])
def has_outstanding_pages(self, site):
results_iter = self.r.table("pages").between(

2
setup.py
View file

@ -21,7 +21,7 @@ import setuptools
setuptools.setup(
name='brozzler',
version='1.1.dev34',
version='1.1.dev35',
description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler',
author='Noah Levitt',