Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 23:56:34 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'master' into qa

Browse Source 
* master:
  peg to working doublethink
  Use disk cache params only on Chrome.start
  Remove stale comment
  Improve disk cache options
  Add disk cache options to Chrome
  update (C)
This commit is contained in:
Noah Levitt 2019-03-14 20:06:12 +00:00
commit c686fc7443
2 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
brozzler/chrome.py
View File

@ -134,7 +134,8 @@ class Chrome:
cookie_location, exc_info=True)
return cookie_db
def start(self, proxy=None, cookie_db=None):
def start(self, proxy=None, cookie_db=None, disk_cache_dir=None,
disk_cache_size=None):
'''
Starts chrome/chromium process.
@ -144,7 +145,10 @@ class Chrome:
which, if supplied, will be written to
{chrome_user_data_dir}/Default/Cookies before running the
browser (default None)
disk_cache_dir: use directory for disk cache. The default location
is inside `self._home_tmpdir` (default None).
disk_cache_size: Forces the maximum disk space to be used by the disk
cache, in bytes. (default None)
Returns:
websocket url to chrome window with about:blank loaded
'''
@ -166,12 +170,17 @@ class Chrome:
'--disable-background-networking',
'--disable-renderer-backgrounding', '--disable-hang-monitor',
'--disable-background-timer-throttling', '--mute-audio',
'--disable-web-sockets', '--disable-cache',
'--disable-web-sockets',
'--window-size=1100,900', '--no-default-browser-check',
'--disable-first-run-ui', '--no-first-run',
'--homepage=about:blank', '--disable-direct-npapi-requests',
'--disable-web-security', '--disable-notifications',
'--disable-extensions', '--disable-save-password-bubble']
if disk_cache_dir:
chrome_args.append('--disk-cache-dir=%s' % disk_cache_dir)
if disk_cache_size:
chrome_args.append('--disk-cache-size=%s' % disk_cache_size)
if self.ignore_cert_errors:
chrome_args.append('--ignore-certificate-errors')
if proxy:

4
setup.py
View File

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup(
name='brozzler',
version='1.5.dev320',
version='1.5.0',
description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler',
author='Noah Levitt',
@ -71,7 +71,7 @@ setuptools.setup(
'websocket-client>=0.39.0,<=0.48.0',
'pillow>=5.2.0',
'urlcanon>=0.1.dev23',
'doublethink>=0.2.0.dev90',
'doublethink>=0.2.0',
'rethinkdb>=2.3',
'cerberus>=1.0.1',
'jinja2>=2.10',