Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 23:56:34 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'optimizes-rethinkdb-load-query' into qa

Browse Source
This commit is contained in:
James Kafader 2020-03-11 19:21:45 -07:00
commit 8a4743bc8b
1 changed files with 1 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
brozzler/worker.py
View File

@ -86,21 +86,11 @@ class BrozzlerWorker:
['ACTIVE', r.minval], ['ACTIVE', r.maxval],
index='sites_last_disclaimed').group('proxy').count()
# returns results like
# [ {
# "group": "wbgrp-svc030.us.archive.org:8000" ,
# "reduction": 148
# } ,
# {
# "group": "wbgrp-svc030.us.archive.org:8001" ,
# "reduction": 145
# }]
proxy_list = list(reql.run())
# convert to structure like:
# {
# "wbgrp-svc030.us.archive.org:8000": 148,
# "wbgrp-svc030.us.archive.org:8001": 145
# }
proxy_scoreboard = {proxy['group']: proxy['reduction'] for proxy in proxy_list}
proxy_scoreboard = list(reql.run())
for warcprox in warcproxes:
address = '%s:%s' % (warcprox['host'], warcprox['port'])
warcprox['assigned_sites'] = proxy_scoreboard.get('address', 0)