Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 00:29:53 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

avoid use of __double_underscore member variables because they're special https://shahriar.svbtle.com/underscores-in-python

Browse Source
This commit is contained in:
Noah Levitt 2016-10-18 18:57:11 -07:00
parent becd832ea3
commit 21891476c4
2 changed files with 12 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
brozzler/worker.py
View File

@ -100,8 +100,8 @@ class BrozzlerWorker:
self._max_browsers = max_browsers
# these two settings can be overridden by the job/site configuration
self.__proxy = proxy
self.__enable_warcprox_features = enable_warcprox_features
self._default_proxy = proxy
self._default_enable_warcprox_features = enable_warcprox_features
self._browser_pool = brozzler.browser.BrowserPool(max_browsers,
chrome_exe=chrome_exe, ignore_cert_errors=True)
@ -113,18 +113,17 @@ class BrozzlerWorker:
def _proxy(self, site):
if site.proxy:
return site.proxy
elif self.__proxy:
return self.__proxy
elif self._default_proxy:
return self._default_proxy
elif self._service_registry and (
site.enable_warcprox_features or
self.__enable_warcprox_features):
warcprox_service = self._service_registry.available_service('warcprox')
site.proxy = '%s:%s' % (warcprox_service['host'],
warcprox_service['port'])
site.enable_warcprox_features
or self._default_enable_warcprox_features):
svc = self._service_registry.available_service('warcprox')
site.proxy = '%s:%s' % (svc['host'], svc['port'])
self._frontier.update_site(site)
self.logger.info(
'chose warcprox %s from service registry for site %s',
site.proxy, site)
'chose warcprox instance %s from service registry for %s',
repr(site.proxy), site)
return site.proxy
return None
@ -133,7 +132,7 @@ class BrozzlerWorker:
if site.enable_warcprox_features is not None:
return site.enable_warcprox_features
else:
return self.__enable_warcprox_features
return self._default_enable_warcprox_features
def _youtube_dl(self, destdir, site):
ydl_opts = {

2
setup.py
View File

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup(
name='brozzler',
version='1.1b7.dev104',
version='1.1b7.dev105',
description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler',
author='Noah Levitt',