Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 23:56:34 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

logging tweaks

Browse Source
This commit is contained in:
Noah Levitt 2015-11-12 02:58:26 +00:00
parent 5597b4cf1a
commit 38dec97e19
1 changed files with 1 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
brozzler/frontier.py
View File

@ -37,7 +37,7 @@ class RethinkDbFrontier:
self.r.table_create("jobs", shards=self.shards, replicas=self.replicas).run()
def _vet_result(self, result, **kwargs):
self.logger.debug("vetting expected=%s result=%s", kwargs, result)
# self.logger.debug("vetting expected=%s result=%s", kwargs, result)
# {'replaced': 0, 'errors': 0, 'skipped': 0, 'inserted': 1, 'deleted': 0, 'generated_keys': ['292859c1-4926-4b27-9d87-b2c367667058'], 'unchanged': 0}
for k in ["replaced", "errors", "skipped", "inserted", "deleted", "unchanged"]:
if k in kwargs:
@ -115,12 +115,10 @@ class RethinkDbFrontier:
return False
def claim_page(self, site, worker_id):
# import pdb; pdb.set_trace()
result = (self.r.table("pages")
.between([site.id, 0, False, self.r.minval], [site.id, 0, False, self.r.maxval], index="priority_by_site")
.order_by(index=rethinkdb.desc("priority_by_site")).limit(1)
.update({"claimed":True,"last_claimed_by":worker_id},return_changes=True)).run()
self.logger.info("query returned %s", result)
self._vet_result(result, replaced=[0,1])
if result["replaced"] == 1:
return brozzler.Page(**result["changes"][0]["new_val"])