Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-09-18 19:54:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

make umbra amenable to clustering by using a pool of n browsers and removing the browser-clientId affinity (not useful currently since we start a fresh browser instance for each page browsed), and set prefetch_count=1 on amqp consumers to round-robin incoming urls among umbra instances

Browse source
This commit is contained in:
Noah Levitt 2014-05-23 21:59:34 -07:00
parent 8d269f4c56
commit 2c4ba005b5
4 changed files with 94 additions and 70 deletions
Download patch file Download diff file Expand all files Collapse all files

9
bin/umbra
View file

@ -8,7 +8,7 @@ import umbra
import sys
import signal
import os
import umbra.controller
import umbra
if __name__=="__main__":
import faulthandler
@ -38,18 +38,19 @@ if __name__=="__main__":
logging.basicConfig(stream=sys.stdout, level=args.log_level,
format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')
controller = umbra.controller.AmqpBrowserController(args.amqp_url,
args.chrome_exe, args.browser_wait,
umbra = umbra.Umbra(args.amqp_url, args.chrome_exe, args.browser_wait,
max_active_browsers=int(args.max_browsers),
exchange_name=args.amqp_exchange, queue_name=args.amqp_queue,
routing_key=args.amqp_routing_key)
umbra.start()
try:
while True:
time.sleep(0.5)
except:
pass
finally:
controller.shutdown()
umbra.shutdown()