Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-09 15:02:28 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

log ports chose for browsers, and give threads nice names to make logs easier to understand

Browse source
This commit is contained in:
Noah Levitt 2014-05-23 22:30:25 -07:00
parent 2c4ba005b5
commit b67d9fadf0
2 changed files with 14 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

10
umbra/browser.py
View file

@ -16,6 +16,8 @@ import socket
from umbra.behaviors import Behavior from umbra.behaviors import Behavior
class BrowserPool: class BrowserPool:
logger = logging.getLogger(__module__ + "." + __qualname__)
def __init__(self, size=3, chrome_exe='chromium-browser', chrome_wait=60): def __init__(self, size=3, chrome_exe='chromium-browser', chrome_wait=60):
self._available = set() self._available = set()
@ -26,6 +28,8 @@ class BrowserPool:
self._lock = threading.Lock() self._lock = threading.Lock()
self.logger.info("browser ports: {}".format([browser.chrome_port for (browser, port_holder) in self._available]))
def _grab_random_port(self): def _grab_random_port(self):
"""Returns socket bound to some port.""" """Returns socket bound to some port."""
sock = socket.socket() sock = socket.socket()
@ -80,7 +84,11 @@ class Browser:
self.websock = websocket.WebSocketApp(websocket_url, self.websock = websocket.WebSocketApp(websocket_url,
on_open=self._visit_page, on_open=self._visit_page,
on_message=self._handle_message) on_message=self._handle_message)
websock_thread = threading.Thread(target=self.websock.run_forever, kwargs={'ping_timeout':0.5})
import random
threadName = "WebsockThread{}-{}".format(self.chrome_port,
''.join((random.choice('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789') for _ in range(6))))
websock_thread = threading.Thread(target=self.websock.run_forever, name=threadName, kwargs={'ping_timeout':0.5})
websock_thread.start() websock_thread.start()
start = time.time() start = time.time()

7
umbra/controller.py
View file

@ -65,7 +65,7 @@ class AmqpBrowserController:
self._producer_conn = kombu.Connection(self.amqp_url) self._producer_conn = kombu.Connection(self.amqp_url)
self._producer = self._producer_conn.Producer(serializer='json') self._producer = self._producer_conn.Producer(serializer='json')
self._amqp_thread = threading.Thread(target=self._consume_amqp) self._amqp_thread = threading.Thread(target=self._consume_amqp, name='AmqpConsumerThread')
self._amqp_stop = threading.Event() self._amqp_stop = threading.Event()
self._amqp_thread.start() self._amqp_thread.start()
@ -139,5 +139,8 @@ class AmqpBrowserController:
browser.browse_page(url, on_request=on_request) browser.browse_page(url, on_request=on_request)
self._browser_pool.release(browser) self._browser_pool.release(browser)
threading.Thread(target=browse_page_async).start() import random
threadName = "BrowsingThread{}-{}".format(browser.chrome_port,
''.join((random.choice('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789') for _ in range(6))))
threading.Thread(target=browse_page_async, name=threadName).start()