Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-25 17:11:16 -05:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'logging' into qa

Browse Source
This commit is contained in:
Misty De Méo 2025-02-24 11:42:05 -08:00
commit 3cdb62ab81
5 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
brozzler/__init__.py
View File

@ -180,7 +180,7 @@ class ThreadExceptionGate:
"exception already pending for thread, discarding", "exception already pending for thread, discarding",
pending_exception=self.pending_exception, pending_exception=self.pending_exception,
thread=self.thread, thread=self.thread,
exception=e, discarded_exception=e,
) )
else: else:
self.pending_exception = e self.pending_exception = e

2
brozzler/browser.py
View File

@ -293,7 +293,7 @@ class WebsockReceiverThread(threading.Thread):
message["params"]["message"]["text"], message["params"]["message"]["text"],
) )
elif message["method"] == "Runtime.exceptionThrown": elif message["method"] == "Runtime.exceptionThrown":
self.logger.debug("uncaught exception", exception=message) self.logger.debug("uncaught exception", message=message)
elif message["method"] == "Page.javascriptDialogOpening": elif message["method"] == "Page.javascriptDialogOpening":
self._javascript_dialog_opening(message) self._javascript_dialog_opening(message)
elif ( elif (

2
brozzler/cli.py
View File

@ -418,7 +418,7 @@ def brozzle_page(argv=None):
on_screenshot=on_screenshot, on_screenshot=on_screenshot,
enable_youtube_dl=not args.skip_youtube_dl, enable_youtube_dl=not args.skip_youtube_dl,
) )
logger.info("outlinks: \n\t%s", "\n\t".join(sorted(outlinks))) logger.info("outlinks", outlinks=sorted(outlinks))
except brozzler.ReachedLimit as e: except brozzler.ReachedLimit as e:
logger.exception("reached limit") logger.exception("reached limit")
except brozzler.PageInterstitialShown as e: except brozzler.PageInterstitialShown as e:

2
brozzler/robots.py
View File

@ -122,6 +122,6 @@ def is_permitted_by_robots(site, url, proxy=None):
structlog.get_logger(logger_name=__name__).warning( structlog.get_logger(logger_name=__name__).warning(
"returning true (permitted) after problem fetching " "robots.txt", "returning true (permitted) after problem fetching " "robots.txt",
url=url, url=url,
exception=e, raised_exception=e,
) )
return True return True

12
tests/test_cluster.py
View File

@ -73,11 +73,19 @@ def stop_service(service):
def httpd(request): def httpd(request):
class RequestHandler(http.server.SimpleHTTPRequestHandler): class RequestHandler(http.server.SimpleHTTPRequestHandler):
def do_POST(self): def do_POST(self):
logger.info("\n%s\n%s", self.requestline, self.headers) logger.info(
"RequestHandler.do_POST",
requestline=self.requestline,
headers=self.headers,
)
self.do_GET() self.do_GET()
def do_GET(self): def do_GET(self):
logger.info("\n%s\n%s", self.requestline, self.headers) logger.info(
"RequestHandler.do_GET",
requestline=self.requestline,
headers=self.headers,
)
if self.path == "/site5/redirect/": if self.path == "/site5/redirect/":
self.send_response(303, "See other") self.send_response(303, "See other")
self.send_header("Connection", "close") self.send_header("Connection", "close")