diff --git a/brozzler/__init__.py b/brozzler/__init__.py index ea15dfd..edf7446 100644 --- a/brozzler/__init__.py +++ b/brozzler/__init__.py @@ -77,8 +77,20 @@ def fixup(url, hash_strip=False): hurl.host = hurl.host.encode('idna').decode('ascii').lower() return hurl.getURLString() -# logging level more fine-grained than logging.DEBUG==10 +# monkey-patch log level TRACE TRACE = 5 +import logging, sys +def _logging_trace(msg, *args, **kwargs): + if len(logging.root.handlers) == 0: + basicConfig() + logging.root.trace(msg, *args, **kwargs) +def _logger_trace(self, msg, *args, **kwargs): + if self.isEnabledFor(TRACE): + self._log(TRACE, msg, args, **kwargs) +logging.trace = _logging_trace +logging.Logger.trace = _logger_trace +logging._levelToName[TRACE] = 'TRACE' +logging._nameToLevel['TRACE'] = TRACE _behaviors = None def behaviors(): diff --git a/brozzler/chrome.py b/brozzler/chrome.py index 4ca476b..e5ef311 100644 --- a/brozzler/chrome.py +++ b/brozzler/chrome.py @@ -231,11 +231,11 @@ class Chrome: b'CERT_PKIXVerifyCert for [^ ]* failed|' b'^ALSA lib|ERROR:gl_surface_glx.cc|' b'ERROR:gpu_child_thread.cc', buf): - logging.log( - brozzler.TRACE, 'chrome pid %s STDOUT %s', + self.logger.trace( + 'chrome pid %s STDOUT %s', self.chrome_process.pid, buf) else: - logging.debug( + self.logger.debug( 'chrome pid %s STDOUT %s', self.chrome_process.pid, buf) @@ -246,15 +246,15 @@ class Chrome: b'CERT_PKIXVerifyCert for [^ ]* failed|' b'^ALSA lib|ERROR:gl_surface_glx.cc|' b'ERROR:gpu_child_thread.cc', buf): - logging.log( - brozzler.TRACE, 'chrome pid %s STDOUT %s', + self.logger.trace( + 'chrome pid %s STDOUT %s', self.chrome_process.pid, buf) else: - logging.debug( + self.logger.debug( 'chrome pid %s STDERR %s', self.chrome_process.pid, buf) except: - logging.error('unexpected exception', exc_info=True) + self.logger.error('unexpected exception', exc_info=True) def stop(self): if not self.chrome_process or self._shutdown.is_set(): diff --git a/brozzler/cli.py b/brozzler/cli.py index fcd5904..9e9545a 100644 --- a/brozzler/cli.py +++ b/brozzler/cli.py @@ -91,8 +91,7 @@ def _add_proxy_options(arg_parser): def configure_logging(args): logging.basicConfig( - stream=sys.stderr, level=args.log_level, - format=( + stream=sys.stderr, level=args.log_level, format=( '%(asctime)s %(process)d %(levelname)s %(threadName)s ' '%(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')) logging.getLogger('requests.packages.urllib3').setLevel(logging.WARN) diff --git a/brozzler/frontier.py b/brozzler/frontier.py index 43bf10b..8bf8e8f 100644 --- a/brozzler/frontier.py +++ b/brozzler/frontier.py @@ -119,7 +119,7 @@ class RethinkDbFrontier: self._vet_result(result, replaced=[0,1], unchanged=[0,1]) def update_page(self, page): - self.logger.debug("updating 'pages' table entry %s", page) + self.logger.trace("updating 'pages' table entry %s", page) result = self.r.table("pages").get(page.id).replace(page.to_dict()).run() self._vet_result(result, replaced=[0,1], unchanged=[0,1]) diff --git a/brozzler/worker.py b/brozzler/worker.py index f20ed86..7eb7d22 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -383,9 +383,8 @@ class BrozzlerWorker: try: self.status_info = self._service_registry.heartbeat(status_info) - self.logger.log( - brozzler.TRACE, "status in service registry: %s", - self.status_info) + self.logger.trace( + "status in service registry: %s", self.status_info) except rethinkdb.ReqlError as e: self.logger.error( "failed to send heartbeat and update service registry " diff --git a/setup.py b/setup.py index 249d026..0f3b228 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.1b9.dev179', + version='1.1b9.dev180', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt',