Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-01 04:56:29 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

ugh, avoid infinite recursion

Browse source
This commit is contained in:
Noah Levitt 2017-03-22 15:53:58 -07:00
parent 34bb64297f
commit 4ba25db684
2 changed files with 3 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

7
brozzler/worker.py
View file

@ -138,10 +138,7 @@ class BrozzlerWorker:
return site.proxy
return None
def _enable_warcprox_features(self, site):
if not self._proxy(site):
return False
if site.enable_warcprox_features is not None:
return site.enable_warcprox_features
else:
@ -227,7 +224,7 @@ class BrozzlerWorker:
info = ydl.extract_info(page.url)
self._remember_videos(page, ydl.brozzler_spy)
# logging.info('XXX %s', json.dumps(info))
if self._enable_warcprox_features(site):
if self._proxy(site) and self._enable_warcprox_features(site):
info_json = json.dumps(info, sort_keys=True, indent=4)
self.logger.info(
"sending WARCPROX_WRITE_RECORD request to warcprox "
@ -306,7 +303,7 @@ class BrozzlerWorker:
def _on_screenshot(screenshot_png):
if on_screenshot:
on_screenshot(screenshot_png)
if self._enable_warcprox_features(site):
if self._proxy(site) and self._enable_warcprox_features(site):
self.logger.info(
"sending WARCPROX_WRITE_RECORD request to %s with "
"screenshot for %s", self._proxy(site), page)

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup(
name='brozzler',
version='1.1b10.dev216',
version='1.1b10.dev217',
description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler',
author='Noah Levitt',