Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-27 08:10:45 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

--trace level logging

Browse source
This commit is contained in:
Noah Levitt 2016-06-29 18:29:45 -05:00
parent d04c5a31cc
commit 479713e25b
5 changed files with 11 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

3
brozzler/__init__.py
View file

@ -64,6 +64,9 @@ class BaseDictable:
def __repr__(self): def __repr__(self):
return "{}(**{})".format(self.__class__.__name__, self.to_dict()) return "{}(**{})".format(self.__class__.__name__, self.to_dict())
# logging level more fine-grained than logging.DEBUG==10
TRACE = 5
from brozzler.site import Page, Site from brozzler.site import Page, Site
from brozzler.worker import BrozzlerWorker from brozzler.worker import BrozzlerWorker
from brozzler.robots import is_permitted_by_robots from brozzler.robots import is_permitted_by_robots

2
brozzler/browser.py
View file

@ -464,7 +464,7 @@ class Chrome:
# returns websocket url to chrome window with about:blank loaded # returns websocket url to chrome window with about:blank loaded
def start(self): def start(self):
timeout_sec = 600 timeout_sec = 20
new_env = os.environ.copy() new_env = os.environ.copy()
new_env["HOME"] = self.user_home_dir new_env["HOME"] = self.user_home_dir
chrome_args = [ chrome_args = [

3
brozzler/cli.py
View file

@ -41,6 +41,9 @@ def _add_common_options(arg_parser):
arg_parser.add_argument( arg_parser.add_argument(
'-v', '--verbose', dest='log_level', '-v', '--verbose', dest='log_level',
action='store_const', default=logging.INFO, const=logging.DEBUG) action='store_const', default=logging.INFO, const=logging.DEBUG)
arg_parser.add_argument(
'--trace', dest='log_level',
action='store_const', default=logging.INFO, const=brozzler.TRACE)
arg_parser.add_argument( arg_parser.add_argument(
'--version', action='version', '--version', action='version',
version='brozzler %s - %s' % ( version='brozzler %s - %s' % (

5
brozzler/worker.py
View file

@ -326,8 +326,9 @@ class BrozzlerWorker:
try: try:
self.status_info = self._service_registry.heartbeat(status_info) self.status_info = self._service_registry.heartbeat(status_info)
self.logger.debug( self.logger.log(
"status in service registry: %s", self.status_info) brozzler.TRACE, "status in service registry: %s",
self.status_info)
except rethinkdb.ReqlError as e: except rethinkdb.ReqlError as e:
self.logger.error( self.logger.error(
"failed to send heartbeat and update service registry " "failed to send heartbeat and update service registry "

2
setup.py
View file

@ -21,7 +21,7 @@ import setuptools
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1.dev31', version='1.1.dev32',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',