Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-08 06:22:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into qa

Browse source 
* master:
  peg to working doublethink
  Use disk cache params only on Chrome.start
  Remove stale comment
  Improve disk cache options
  Add disk cache options to Chrome
  update (C)
This commit is contained in:
Noah Levitt 2019-03-14 20:06:12 +00:00
commit c686fc7443
2 changed files with 14 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

15
brozzler/chrome.py
View file

@ -134,7 +134,8 @@ class Chrome:
cookie_location, exc_info=True) cookie_location, exc_info=True)
return cookie_db return cookie_db
def start(self, proxy=None, cookie_db=None): def start(self, proxy=None, cookie_db=None, disk_cache_dir=None,
disk_cache_size=None):
''' '''
Starts chrome/chromium process. Starts chrome/chromium process.
@ -144,7 +145,10 @@ class Chrome:
which, if supplied, will be written to which, if supplied, will be written to
{chrome_user_data_dir}/Default/Cookies before running the {chrome_user_data_dir}/Default/Cookies before running the
browser (default None) browser (default None)
disk_cache_dir: use directory for disk cache. The default location
is inside `self._home_tmpdir` (default None).
disk_cache_size: Forces the maximum disk space to be used by the disk
cache, in bytes. (default None)
Returns: Returns:
websocket url to chrome window with about:blank loaded websocket url to chrome window with about:blank loaded
''' '''
@ -166,12 +170,17 @@ class Chrome:
'--disable-background-networking', '--disable-background-networking',
'--disable-renderer-backgrounding', '--disable-hang-monitor', '--disable-renderer-backgrounding', '--disable-hang-monitor',
'--disable-background-timer-throttling', '--mute-audio', '--disable-background-timer-throttling', '--mute-audio',
'--disable-web-sockets', '--disable-cache', '--disable-web-sockets',
'--window-size=1100,900', '--no-default-browser-check', '--window-size=1100,900', '--no-default-browser-check',
'--disable-first-run-ui', '--no-first-run', '--disable-first-run-ui', '--no-first-run',
'--homepage=about:blank', '--disable-direct-npapi-requests', '--homepage=about:blank', '--disable-direct-npapi-requests',
'--disable-web-security', '--disable-notifications', '--disable-web-security', '--disable-notifications',
'--disable-extensions', '--disable-save-password-bubble'] '--disable-extensions', '--disable-save-password-bubble']
if disk_cache_dir:
chrome_args.append('--disk-cache-dir=%s' % disk_cache_dir)
if disk_cache_size:
chrome_args.append('--disk-cache-size=%s' % disk_cache_size)
if self.ignore_cert_errors: if self.ignore_cert_errors:
chrome_args.append('--ignore-certificate-errors') chrome_args.append('--ignore-certificate-errors')
if proxy: if proxy:

4
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.5.dev320', version='1.5.0',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',
@ -71,7 +71,7 @@ setuptools.setup(
'websocket-client>=0.39.0,<=0.48.0', 'websocket-client>=0.39.0,<=0.48.0',
'pillow>=5.2.0', 'pillow>=5.2.0',
'urlcanon>=0.1.dev23', 'urlcanon>=0.1.dev23',
'doublethink>=0.2.0.dev90', 'doublethink>=0.2.0',
'rethinkdb>=2.3', 'rethinkdb>=2.3',
'cerberus>=1.0.1', 'cerberus>=1.0.1',
'jinja2>=2.10', 'jinja2>=2.10',