Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-19 07:15:52 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

better error handling for _fetch_url

Browse Source
This commit is contained in:
Barbara Miller 2025-02-13 14:21:24 -08:00
parent 5fccdd83e3
commit 53a1869def
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
brozzler/worker.py
View File

@ -503,7 +503,11 @@ class BrozzlerWorker:
timeout=self.FETCH_URL_TIMEOUT,
)
except requests.exceptions.Timeout as e:
self.logger.warning("Timed out fetching %s: %s", page.url, e)
self.logger.warning("Timed out fetching %s", url)
if 'archive.org' in e:
raise brozzler.ProxyError("proxy error fetching %s" % url) from e
else:
raise brozzler.PageConnectionError(timeout error fetching %s" % url) from e
except requests.exceptions.ProxyError as e:
raise brozzler.ProxyError("proxy error fetching %s" % url) from e
except requests.exceptions.RequestException as e: