Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-07-13 18:19:27 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

catch and log more exceptions on fetch_url error

Browse source
This commit is contained in:
Barbara Miller 2025-02-11 12:51:26 -08:00
parent 561e0803c6
commit 430c0daf39
1 changed files with 2 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
brozzler/worker.py
View file

@ -506,6 +506,8 @@ class BrozzlerWorker:
self.logger.warning("Timed out fetching %s: %s", page.url, e)
except requests.exceptions.ProxyError as e:
raise brozzler.ProxyError("proxy error fetching %s" % url) from e
except requests.exceptions.RequestException as e:
self.logger.warning("Failed to get headers for %s: %s", page.url, e)
def brozzle_site(self, browser, site):
try: