Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-19 12:24:20 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fail quickly if browser dies at startup

Browse source 
instead of trying to retrieve /json for 600 seconds
This commit is contained in:
Noah Levitt 2018-11-01 15:57:03 -07:00
parent 1073431f76
commit 15610fa990
2 changed files with 17 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

25
brozzler/chrome.py
View file

@ -219,16 +219,23 @@ class Chrome:
'of %d seconds): %s', json_url, timeout_sec, e) 'of %d seconds): %s', json_url, timeout_sec, e)
self._last_warning = time.time() self._last_warning = time.time()
finally: finally:
if time.time() - self._start > timeout_sec: e = None
self.logger.error( if self.chrome_process:
'killing chrome, failed to retrieve %s after % ' if time.time() - self._start > timeout_sec:
'seconds', json_url, time.time() - self._start) e = Exception(
self.stop() 'killing chrome, failed to retrieve %s after '
raise Exception( '%s seconds' % (
'killed chrome, failed to retrieve %s after %s ' json_url, time.time() - self._start))
'seconds' % (json_url, time.time() - self._start)) elif self.chrome_process.poll() is not None:
e = Exception(
'chrome process died with status %s' % self.chrome_process.poll())
else:
time.sleep(0.5)
else: else:
time.sleep(0.5) e = Exception('??? self.chrome_process is not set ???')
if e:
self.stop()
raise e
def _read_stderr_stdout(self): def _read_stderr_stdout(self):
# XXX select doesn't work on windows # XXX select doesn't work on windows

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.5.dev312', version='1.5.dev313',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',