Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-07-24 15:25:40 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

maybe fewer warcproxes

Browse source
This commit is contained in:
Barbara Miller 2020-03-11 14:08:34 -07:00
parent d190122a6d
commit f8f7aa1dca
1 changed files with 4 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

5
brozzler/worker.py
View file

@ -82,8 +82,11 @@ class BrozzlerWorker:
if not warcproxes: if not warcproxes:
return None return None
warcproxes.sort(key=lambda warcprox: (warcprox['load'])) warcproxes.sort(key=lambda warcprox: (warcprox['load']))
num_choices = 5
if len(warcproxes) < num_choices:
num_choices = len(warcproxes)
# XXX make this heuristic more advanced? # XXX make this heuristic more advanced?
return random.choice(warcproxes[0:5]) return random.choice(warcproxes[0:num_choices])
def _proxy_for(self, site): def _proxy_for(self, site):
if self._proxy: if self._proxy: