wait for a browser to become available and start it up before reading the next url from amqp; ack the message only after completing the browsing process successfully, and requeue if it's not successful; some refactoring to make the timing work for this

This commit is contained in:
Noah Levitt 2014-06-09 13:15:05 -07:00
parent e3c23a0f2b
commit a78e60f1da
4 changed files with 145 additions and 78 deletions

View file

@ -5,7 +5,8 @@ import logging
import time
import threading
import kombu
from umbra.browser import BrowserPool
import socket
from umbra.browser import BrowserPool, BrowsingException
class AmqpBrowserController:
"""
@ -56,6 +57,9 @@ class AmqpBrowserController:
chrome_exe=chrome_exe, chrome_wait=browser_wait)
def start(self):
self._browsing_threads = set()
self._browsing_threads_lock = threading.Lock()
self._exchange = kombu.Exchange(name=self.exchange_name, type='direct',
durable=True)
@ -65,20 +69,19 @@ class AmqpBrowserController:
self._producer_conn = kombu.Connection(self.amqp_url)
self._producer = self._producer_conn.Producer(serializer='json')
self._amqp_thread = threading.Thread(target=self._consume_amqp, name='AmqpConsumerThread')
self._amqp_stop = threading.Event()
self._amqp_thread.start()
self._consumer_thread = threading.Thread(target=self._consume_amqp, name='AmqpConsumerThread')
self._consumer_stop = threading.Event()
self._consumer_thread.start()
def shutdown(self):
self.logger.info("shutting down amqp consumer {}".format(self.amqp_url))
self._amqp_stop.set()
self._amqp_thread.join()
# with self._producer_lock:
# self._producer_conn.close()
# self._producer_conn = None
self._consumer_stop.set()
self._consumer_thread.join()
def shutdown_now(self):
self._consumer_stop.set()
self._browser_pool.shutdown_now()
self._consumer_thread.join()
def _consume_amqp(self):
# XXX https://webarchive.jira.com/browse/ARI-3811
@ -92,30 +95,51 @@ class AmqpBrowserController:
url_queue = kombu.Queue(self.queue_name, routing_key=self.routing_key,
exchange=self._exchange)
while not self._amqp_stop.is_set():
while not self._consumer_stop.is_set():
try:
self.logger.info("connecting to amqp exchange={} at {}".format(self._exchange.name, self.amqp_url))
with kombu.Connection(self.amqp_url) as conn:
conn_opened = time.time()
with conn.Consumer(url_queue) as consumer:
consumer.qos(prefetch_count=1)
while (not self._amqp_stop.is_set() and time.time() - conn_opened < RECONNECT_AFTER_SECONDS):
import socket
browser = None
while not self._consumer_stop.is_set() and time.time() - conn_opened < RECONNECT_AFTER_SECONDS:
try:
browser = self._browser_pool.acquire() # raises KeyError if none available
browser.start()
consumer.callbacks = [self._make_callback(browser)]
conn.drain_events(timeout=0.5)
while True:
try:
conn.drain_events(timeout=0.5)
break # out of "while True" to acquire another browser
except socket.timeout:
pass
if self._consumer_stop.is_set() or time.time() - conn_opened >= RECONNECT_AFTER_SECONDS:
browser.stop()
self._browser_pool.release(browser)
break
except KeyError:
# no browsers available
time.sleep(0.5)
except:
self.logger.critical("problem with browser initialization", exc_info=True)
time.sleep(0.5)
finally:
consumer.callbacks = None
# browser startup is a heavy operation, so do
# it once every 5 seconds at most
time.sleep(5)
except KeyError:
# thrown by self._browser_pool.acquire() - no browsers available
time.sleep(0.5)
except socket.timeout:
# thrown by conn.drain_events(timeout=0.5) - no urls in the queue
self._browser_pool.release(browser)
# need to wait for browsers to finish here, before closing
# the amqp connection, because they use it to do
# message.ack() after they finish browsing a page
self.logger.info("waiting for browsing threads to finish")
while True:
with self._browsing_threads_lock:
if len(self._browsing_threads) == 0:
break
time.sleep(0.5)
self.logger.info("browsing threads finished")
except BaseException as e:
self.logger.error("caught exception {}".format(e), exc_info=True)
@ -124,11 +148,10 @@ class AmqpBrowserController:
def _make_callback(self, browser):
def callback(body, message):
self._browse_page(browser, body['clientId'], body['url'], body['metadata'])
message.ack()
self._start_browsing_page(browser, message, body['clientId'], body['url'], body['metadata'])
return callback
def _browse_page(self, browser, client_id, url, parent_url_metadata):
def _start_browsing_page(self, browser, message, client_id, url, parent_url_metadata):
def on_request(chrome_msg):
payload = chrome_msg['params']['request']
payload['parentUrl'] = url
@ -142,12 +165,25 @@ class AmqpBrowserController:
self.logger.info('browser={} client_id={} url={}'.format(browser, client_id, url))
try:
browser.browse_page(url, on_request=on_request)
self._browser_pool.release(browser)
message.ack()
except BrowsingException as e:
self.logger.warn("browsing did not complete normally, requeuing url {} - {}".format(url, e))
message.requeue()
except:
self.logger.critical("problem browsing page, may have lost browser process", exc_info=True)
self.logger.critical("problem browsing page, requeuing url {}, may have lost browser process".format(url), exc_info=True)
message.requeue()
finally:
browser.stop()
self._browser_pool.release(browser)
with self._browsing_threads_lock:
self._browsing_threads.remove(threading.current_thread())
import random
threadName = "BrowsingThread{}-{}".format(browser.chrome_port,
''.join((random.choice('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789') for _ in range(6))))
threading.Thread(target=browse_page_async, name=threadName).start()
th = threading.Thread(target=browse_page_async, name=threadName)
with self._browsing_threads_lock:
self._browsing_threads.add(th)
th.start()