From f8c41c5e8da63b286342472ce5400d7faad2f17b Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 14 Feb 2018 12:29:51 -0800 Subject: [PATCH] bump up timeout waiting for websocket connection We've been seeing some of this: 2018-02-14 20:16:44,011 17816 CRITICAL BrozzlingThread:36444 brozzler.worker.BrozzlerWorker.brozzle_site(worker.py:559) unexpected exception Traceback (most recent call last): File "/opt/brozzler-ve3/lib/python3.5/site-packages/brozzler/worker.py", line 528, in brozzle_site enable_youtube_dl=not self._skip_youtube_dl) File "/opt/brozzler-ve3/lib/python3.5/site-packages/brozzler/worker.py", line 385, in brozzle_page on_request) File "/opt/brozzler-ve3/lib/python3.5/site-packages/brozzler/worker.py", line 447, in _browse_page cookie_db=site.get('cookie_db')) File "/opt/brozzler-ve3/lib/python3.5/site-packages/brozzler/browser.py", line 338, in start self._wait_for(lambda: self.websock_thread.is_open, timeout=10) File "/opt/brozzler-ve3/lib/python3.5/site-packages/brozzler/browser.py", line 311, in _wait_for elapsed, callback)) brozzler.browser.BrowsingTimeout: timed out after 11.1s waiting for: . at 0x7fb2dc772bd8> Mostly at startup. Now that brozzler claims sites in batches for brozzling, we have situations where we start up a whole bunch of browsers at the same time. That's probably why in some cases they are slow to establish the websocket connection. --- brozzler/browser.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/brozzler/browser.py b/brozzler/browser.py index 9f6d07d..4a89795 100644 --- a/brozzler/browser.py +++ b/brozzler/browser.py @@ -335,7 +335,7 @@ class Browser: self.websock, name='WebsockThread:%s' % self.chrome.port) self.websock_thread.start() - self._wait_for(lambda: self.websock_thread.is_open, timeout=10) + self._wait_for(lambda: self.websock_thread.is_open, timeout=30) # tell browser to send us messages we're interested in self.send_to_chrome(method='Network.enable') diff --git a/setup.py b/setup.py index bbfd2bd..8752e6b 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.1b13.dev285', + version='1.1b13.dev286', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt',