Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-23 16:19:49 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

catch and log more exceptions on fetch_url error

Browse Source
This commit is contained in:
Barbara Miller 2025-02-11 12:51:26 -08:00
parent 561e0803c6
commit 430c0daf39
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
brozzler/worker.py
View File

@ -506,6 +506,8 @@ class BrozzlerWorker:
self.logger.warning("Timed out fetching %s: %s", page.url, e) self.logger.warning("Timed out fetching %s: %s", page.url, e)
except requests.exceptions.ProxyError as e: except requests.exceptions.ProxyError as e:
raise brozzler.ProxyError("proxy error fetching %s" % url) from e raise brozzler.ProxyError("proxy error fetching %s" % url) from e
except requests.exceptions.RequestException as e:
self.logger.warning("Failed to get headers for %s: %s", page.url, e)
def brozzle_site(self, browser, site): def brozzle_site(self, browser, site):
try: try: