Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix query to make tests pass?

Browse Source
This commit is contained in:
Noah Levitt 2018-03-02 16:30:35 -08:00
parent f26d711a89
commit 2639d7b991
1 changed files with 10 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
brozzler/frontier.py
View File

@ -98,7 +98,7 @@ class RethinkDbFrontier:
Returns a dictionary that looks like this:
{<job_id>: {'claimed_sites': 2, 'max_claimed_sites': 3}, ...}
'''
# js query: r.db('brozzler').table('sites').between(['ACTIVE', r.minval], ['ACTIVE', r.maxval], {'index': 'sites_last_disclaimed'}).eqJoin('job_id', r.db('brozzler').table('jobs')).group(function(x){return x('right')('id')}).ungroup().map(function(x){return {'job_id': x('group'), 'max_claimed_sites':x('reduction')('right')('max_claimed_sites')(0), 'claimed_sites':x('reduction')('left').filter({'claimed':true}).count()}})
# js query: r.db('brozzler').table('sites').between(['ACTIVE', r.minval], ['ACTIVE', r.maxval], {'index': 'sites_last_disclaimed'}).eqJoin('job_id', r.db('brozzler').table('jobs')).filter(function(x){return x('right').hasFields('max_claimed_sites')}).group(function(x){return x('right')('id')}).ungroup().map(function(x){return {'job_id': x('group'), 'max_claimed_sites':x('reduction')('right')('max_claimed_sites')(0), 'claimed_sites':x('reduction')('left').filter({'claimed':true}).count()}})
# returns results like:
# [{'max_claimed_sites': 2, 'claimed_sites': 0, 'job_id': 1234},
# {'max_claimed_sites': 3, 'claimed_sites': 3, 'job_id': 1235}]
@ -108,6 +108,7 @@ class RethinkDbFrontier:
['ACTIVE', r.minval], ['ACTIVE', r.maxval],
index='sites_last_disclaimed')
.eq_join('job_id', r.db(self.rr.dbname).table('jobs'))
.filter(lambda x: x['right'].has_fields('max_claimed_sites'))
.group(lambda x: x['right']['id'])
.ungroup()
.map(lambda x: {
@ -115,10 +116,14 @@ class RethinkDbFrontier:
'max_claimed_sites': x['reduction']['right']['max_claimed_sites'][0],
'claimed_sites': x['reduction']['left'].filter({'claimed':True}).count()
})).run()
xformed = {d['job_id']: {
'claimed_sites': d['claimed_sites'],
'max_claimed_sites': d['max_claimed_sites']
} for d in results}
xformed = {
d['job_id']: {
'claimed_sites': d['claimed_sites'],
'max_claimed_sites': d['max_claimed_sites']
}
for d in results
}
return xformed
def claim_sites(self, n=1):