Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-20 21:04:24 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

logging tweaks

Browse source
This commit is contained in:
Noah Levitt 2016-07-28 10:19:30 -05:00
parent dd2d8c89e3
commit b2b07b79a9
3 changed files with 33 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

52
brozzler/behaviors.py
View file

@ -1,21 +1,21 @@
# '''
# brozzler/behaviors.py - manages behaviors, which are javascript scripts that brozzler/behaviors.py - manages behaviors, which are javascript scripts that
# run in brozzled web pages run in brozzled web pages
#
# Copyright (C) 2014-2016 Internet Archive Copyright (C) 2014-2016 Internet Archive
#
# Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
# You may obtain a copy of the License at You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0 http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and See the License for the specific language governing permissions and
# limitations under the License. limitations under the License.
# '''
import json import json
import itertools import itertools
@ -111,13 +111,21 @@ class Behavior:
# {'id': 59, 'result': {'result': {'type': 'boolean', 'value': True}, 'wasThrown': False}} # {'id': 59, 'result': {'result': {'type': 'boolean', 'value': True}, 'wasThrown': False}}
# {'id': 59, 'result': {'result': {'type': 'boolean', 'value': False}} # {'id': 59, 'result': {'result': {'type': 'boolean', 'value': False}}
self.waiting_result_msg_ids.remove(chrome_message['id']) self.waiting_result_msg_ids.remove(chrome_message['id'])
if ('result' in chrome_message if ('result' in chrome_message and not (
and not ('wasThrown' in chrome_message['result'] and chrome_message['result']['wasThrown']) 'wasThrown' in chrome_message['result']
and chrome_message['result']['wasThrown'])
and 'result' in chrome_message['result'] and 'result' in chrome_message['result']
and type(chrome_message['result']['result']['value']) == bool): and type(chrome_message['result']['result']['value']) == bool):
self.script_finished = chrome_message['result']['result']['value'] self.script_finished = chrome_message['result']['result']['value']
else: else:
self.logger.error("chrome message doesn't look like a boolean result! {}".format(chrome_message)) # this happens if the behavior script doesn't define
# umbraBehaviorFinished, and I think it can also happen normally
# after the behavior has been sent to the browser but before
# the browser has it fully loaded... in any case the message
# was overwhelming the logs, so I'm bumping it down to debug level
self.logger.debug(
"chrome message doesn't look like a boolean result! %s",
chrome_message)
def notify_of_activity(self): def notify_of_activity(self):
self.last_activity = time.time() self.last_activity = time.time()
@ -127,5 +135,3 @@ if __name__ == "__main__":
format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s') format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')
logger = logging.getLogger('umbra.behaviors') logger = logging.getLogger('umbra.behaviors')
logger.info("custom behaviors: {}".format(Behavior.behaviors())) logger.info("custom behaviors: {}".format(Behavior.behaviors()))

4
brozzler/browser.py
View file

@ -445,7 +445,9 @@ compileOutlinks(window).join(' ');
try: try:
self._handle_message(websock, message) self._handle_message(websock, message)
except: except:
self.logger.error("uncaught exception in _handle_message", exc_info=True) self.logger.error(
"uncaught exception in _handle_message message=%s",
message, exc_info=True)
self.abort_browse_page() self.abort_browse_page()
def _network_request_will_be_sent(self, message): def _network_request_will_be_sent(self, message):

2
setup.py
View file

@ -21,7 +21,7 @@ import setuptools
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b4.dev60', version='1.1b4.dev61',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',