From 4ba25db68484bb8e6a190fbe03f40dcda36d9742 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Wed, 22 Mar 2017 15:53:58 -0700 Subject: [PATCH] ugh, avoid infinite recursion --- brozzler/worker.py | 7 ++----- setup.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index 94c851f..97f23d9 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -138,10 +138,7 @@ class BrozzlerWorker: return site.proxy return None - def _enable_warcprox_features(self, site): - if not self._proxy(site): - return False if site.enable_warcprox_features is not None: return site.enable_warcprox_features else: @@ -227,7 +224,7 @@ class BrozzlerWorker: info = ydl.extract_info(page.url) self._remember_videos(page, ydl.brozzler_spy) # logging.info('XXX %s', json.dumps(info)) - if self._enable_warcprox_features(site): + if self._proxy(site) and self._enable_warcprox_features(site): info_json = json.dumps(info, sort_keys=True, indent=4) self.logger.info( "sending WARCPROX_WRITE_RECORD request to warcprox " @@ -306,7 +303,7 @@ class BrozzlerWorker: def _on_screenshot(screenshot_png): if on_screenshot: on_screenshot(screenshot_png) - if self._enable_warcprox_features(site): + if self._proxy(site) and self._enable_warcprox_features(site): self.logger.info( "sending WARCPROX_WRITE_RECORD request to %s with " "screenshot for %s", self._proxy(site), page) diff --git a/setup.py b/setup.py index 7d6df6c..f04789e 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.1b10.dev216', + version='1.1b10.dev217', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt',