Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-20 12:54:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

avoid use of __double_underscore member variables because they're special https://shahriar.svbtle.com/underscores-in-python

Browse source
This commit is contained in:
Noah Levitt 2016-10-18 18:57:11 -07:00
parent becd832ea3
commit 21891476c4
2 changed files with 12 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

23
brozzler/worker.py
View file

@ -100,8 +100,8 @@ class BrozzlerWorker:
self._max_browsers = max_browsers self._max_browsers = max_browsers
# these two settings can be overridden by the job/site configuration # these two settings can be overridden by the job/site configuration
self.__proxy = proxy self._default_proxy = proxy
self.__enable_warcprox_features = enable_warcprox_features self._default_enable_warcprox_features = enable_warcprox_features
self._browser_pool = brozzler.browser.BrowserPool(max_browsers, self._browser_pool = brozzler.browser.BrowserPool(max_browsers,
chrome_exe=chrome_exe, ignore_cert_errors=True) chrome_exe=chrome_exe, ignore_cert_errors=True)
@ -113,18 +113,17 @@ class BrozzlerWorker:
def _proxy(self, site): def _proxy(self, site):
if site.proxy: if site.proxy:
return site.proxy return site.proxy
elif self.__proxy: elif self._default_proxy:
return self.__proxy return self._default_proxy
elif self._service_registry and ( elif self._service_registry and (
site.enable_warcprox_features or site.enable_warcprox_features
self.__enable_warcprox_features): or self._default_enable_warcprox_features):
warcprox_service = self._service_registry.available_service('warcprox') svc = self._service_registry.available_service('warcprox')
site.proxy = '%s:%s' % (warcprox_service['host'], site.proxy = '%s:%s' % (svc['host'], svc['port'])
warcprox_service['port'])
self._frontier.update_site(site) self._frontier.update_site(site)
self.logger.info( self.logger.info(
'chose warcprox %s from service registry for site %s', 'chose warcprox instance %s from service registry for %s',
site.proxy, site) repr(site.proxy), site)
return site.proxy return site.proxy
return None return None
@ -133,7 +132,7 @@ class BrozzlerWorker:
if site.enable_warcprox_features is not None: if site.enable_warcprox_features is not None:
return site.enable_warcprox_features return site.enable_warcprox_features
else: else:
return self.__enable_warcprox_features return self._default_enable_warcprox_features
def _youtube_dl(self, destdir, site): def _youtube_dl(self, destdir, site):
ydl_opts = { ydl_opts = {

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b7.dev104', version='1.1b7.dev105',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',