Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-01 19:16:15 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

skip active_sites query

Browse source
This commit is contained in:
Barbara Miller 2020-03-11 13:40:37 -07:00
parent c2a1ca018a
commit af39b8cc6f
1 changed files with 1 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

12
brozzler/worker.py
View file

@ -80,17 +80,7 @@ class BrozzlerWorker:
warcproxes = self._service_registry.available_services('warcprox') warcproxes = self._service_registry.available_services('warcprox')
if not warcproxes: if not warcproxes:
return None return None
reql = self._frontier.rr.table('sites').between( warcproxes.sort(key=lambda warcprox: (warcprox['load']))
['ACTIVE', r.minval], ['ACTIVE', r.maxval],
index='sites_last_disclaimed')
active_sites = list(reql.run())
for warcprox in warcproxes:
address = '%s:%s' % (warcprox['host'], warcprox['port'])
warcprox['assigned_sites'] = len([
site for site in active_sites
if 'proxy' in site and site['proxy'] == address])
warcproxes.sort(key=lambda warcprox: (
warcprox['assigned_sites'], warcprox['load']))
# XXX make this heuristic more advanced? # XXX make this heuristic more advanced?
return warcproxes[0] return warcproxes[0]