Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-20 12:54:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

ugh, avoid infinite recursion

Browse source
This commit is contained in:
Noah Levitt 2017-03-22 15:53:58 -07:00
parent 34bb64297f
commit 4ba25db684
2 changed files with 3 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

7
brozzler/worker.py
View file

@ -138,10 +138,7 @@ class BrozzlerWorker:
return site.proxy return site.proxy
return None return None
def _enable_warcprox_features(self, site): def _enable_warcprox_features(self, site):
if not self._proxy(site):
return False
if site.enable_warcprox_features is not None: if site.enable_warcprox_features is not None:
return site.enable_warcprox_features return site.enable_warcprox_features
else: else:
@ -227,7 +224,7 @@ class BrozzlerWorker:
info = ydl.extract_info(page.url) info = ydl.extract_info(page.url)
self._remember_videos(page, ydl.brozzler_spy) self._remember_videos(page, ydl.brozzler_spy)
# logging.info('XXX %s', json.dumps(info)) # logging.info('XXX %s', json.dumps(info))
if self._enable_warcprox_features(site): if self._proxy(site) and self._enable_warcprox_features(site):
info_json = json.dumps(info, sort_keys=True, indent=4) info_json = json.dumps(info, sort_keys=True, indent=4)
self.logger.info( self.logger.info(
"sending WARCPROX_WRITE_RECORD request to warcprox " "sending WARCPROX_WRITE_RECORD request to warcprox "
@ -306,7 +303,7 @@ class BrozzlerWorker:
def _on_screenshot(screenshot_png): def _on_screenshot(screenshot_png):
if on_screenshot: if on_screenshot:
on_screenshot(screenshot_png) on_screenshot(screenshot_png)
if self._enable_warcprox_features(site): if self._proxy(site) and self._enable_warcprox_features(site):
self.logger.info( self.logger.info(
"sending WARCPROX_WRITE_RECORD request to %s with " "sending WARCPROX_WRITE_RECORD request to %s with "
"screenshot for %s", self._proxy(site), page) "screenshot for %s", self._proxy(site), page)

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b10.dev216', version='1.1b10.dev217',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',