Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-07-23 06:50:37 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

robots: convert to structlog

Browse source
This commit is contained in:
Misty De Méo 2025-02-18 14:29:00 -08:00
parent ba0db01d32
commit b33b2fed8c
1 changed files with 5 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

10
brozzler/robots.py
View file

@ -23,12 +23,12 @@ limitations under the License.
""" """
import json import json
import logging
import brozzler import brozzler
import reppy import reppy
import reppy.cache import reppy.cache
import reppy.parser import reppy.parser
import requests import requests
import structlog
__all__ = ["is_permitted_by_robots"] __all__ = ["is_permitted_by_robots"]
@ -119,10 +119,10 @@ def is_permitted_by_robots(site, url, proxy=None):
# reppy has wrapped an exception that we want to bubble up # reppy has wrapped an exception that we want to bubble up
raise brozzler.ProxyError(e) raise brozzler.ProxyError(e)
else: else:
logging.warning( structlog.get_logger().warning(
"returning true (permitted) after problem fetching " "returning true (permitted) after problem fetching "
"robots.txt for %r: %r", "robots.txt",
url, url=url,
e, exception=e,
) )
return True return True