Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 16:49:56 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix case where rethinkdb page already has claimed=True

Browse Source
This commit is contained in:
Noah Levitt 2016-06-29 19:29:18 -05:00
parent ffcf26b6c9
commit 2e687b65fb
2 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
brozzler/frontier.py
View File

@ -197,12 +197,12 @@ class RethinkDbFrontier:
1).update({ 1).update({
"claimed":True, "claimed":True,
"last_claimed_by":worker_id}, "last_claimed_by":worker_id},
return_changes=True).run() return_changes="always").run()
self._vet_result(result, replaced=[0,1]) self._vet_result(result, unchanged=[0,1], replaced=[0,1])
if result["replaced"] == 1: if result["unchanged"] == 0 and result["replaced"] == 0:
return brozzler.Page(**result["changes"][0]["new_val"])
else:
raise brozzler.NothingToClaim raise brozzler.NothingToClaim
else:
return brozzler.Page(**result["changes"][0]["new_val"])
def has_outstanding_pages(self, site): def has_outstanding_pages(self, site):
results_iter = self.r.table("pages").between( results_iter = self.r.table("pages").between(

2
setup.py
View File

@ -21,7 +21,7 @@ import setuptools
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1.dev34', version='1.1.dev35',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',