diff --git a/brozzler/worker.py b/brozzler/worker.py index 13beae3..c2a76be 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -447,12 +447,12 @@ class BrozzlerWorker: def brozzle_site(self, browser, site): try: + start = time.time() page = None self._frontier.honor_stop_request(site) self.logger.info( "brozzling site (proxy=%r) %r", self._proxy_for(site), site) - start = time.time() while time.time() - start < 7 * 60: site.refresh() self._frontier.honor_stop_request(site) @@ -499,7 +499,8 @@ class BrozzlerWorker: except: self.logger.critical("unexpected exception", exc_info=True) finally: - site.active_brozzling_time = (site.active_brozzling_time or 0) + time.time() - start + if start: + site.active_brozzling_time = (site.active_brozzling_time or 0) + time.time() - start self._frontier.disclaim_site(site, page) def _brozzle_site_thread_target(self, browser, site): diff --git a/setup.py b/setup.py index 4db5d2d..f102c15 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.1b12.dev263', + version='1.1b12.dev264', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt',