Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-09 23:13:07 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adds routing_key to queue Queue creation

Browse source
This commit is contained in:
Lauren Ko 2015-07-31 14:15:18 -05:00
parent f254e2eec1
commit d4a783285e
2 changed files with 7 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

5
bin/umbra
View file

@ -26,6 +26,8 @@ if __name__ == "__main__":
help='AMQP exchange name') help='AMQP exchange name')
arg_parser.add_argument('--queue', dest='amqp_queue', default='urls', arg_parser.add_argument('--queue', dest='amqp_queue', default='urls',
help='AMQP queue to consume urls from') help='AMQP queue to consume urls from')
arg_parser.add_argument('--routing-key', dest='amqp_routing_key', default='urls',
help='AMQP routing key to assign to AMQP queue of urls')
arg_parser.add_argument('-n', '--max-browsers', dest='max_browsers', default='1', arg_parser.add_argument('-n', '--max-browsers', dest='max_browsers', default='1',
help='Max number of chrome instances simultaneously browsing pages') help='Max number of chrome instances simultaneously browsing pages')
arg_parser.add_argument('-v', '--verbose', dest='log_level', arg_parser.add_argument('-v', '--verbose', dest='log_level',
@ -41,7 +43,8 @@ if __name__ == "__main__":
controller = umbra.Umbra(args.amqp_url, args.chrome_exe, controller = umbra.Umbra(args.amqp_url, args.chrome_exe,
max_active_browsers=int(args.max_browsers), max_active_browsers=int(args.max_browsers),
exchange_name=args.amqp_exchange, queue_name=args.amqp_queue) exchange_name=args.amqp_exchange, queue_name=args.amqp_queue,
routing_key=args.amqp_routing_key)
def browserdump_str(pp, b): def browserdump_str(pp, b):
x = [] x = []

5
umbra/controller.py
View file

@ -46,10 +46,11 @@ class AmqpBrowserController:
def __init__(self, amqp_url='amqp://guest:guest@localhost:5672/%2f', def __init__(self, amqp_url='amqp://guest:guest@localhost:5672/%2f',
chrome_exe='chromium-browser', max_active_browsers=1, chrome_exe='chromium-browser', max_active_browsers=1,
queue_name='urls', exchange_name='umbra'): queue_name='urls', exchange_name='umbra', routing_key='urls'):
self.amqp_url = amqp_url self.amqp_url = amqp_url
self.queue_name = queue_name self.queue_name = queue_name
self.exchange_name = exchange_name self.exchange_name = exchange_name
self.routing_key = routing_key
self.max_active_browsers = max_active_browsers self.max_active_browsers = max_active_browsers
self._browser_pool = BrowserPool(size=max_active_browsers, chrome_exe=chrome_exe) self._browser_pool = BrowserPool(size=max_active_browsers, chrome_exe=chrome_exe)
@ -153,7 +154,7 @@ class AmqpBrowserController:
# reopen the connection every 2.5 hours # reopen the connection every 2.5 hours
RECONNECT_AFTER_SECONDS = 150 * 60 RECONNECT_AFTER_SECONDS = 150 * 60
url_queue = kombu.Queue(self.queue_name, exchange=self._exchange) url_queue = kombu.Queue(self.queue_name, exchange=self._exchange, routing_key=self.routing_key)
while not self._consumer_stop.is_set(): while not self._consumer_stop.is_set():
try: try: