Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'bmiller/better_fetch_url_timeout_errors' into qa

Browse Source
This commit is contained in:
Barbara Miller 2025-02-14 09:44:24 -08:00
commit 73682a41f7
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
brozzler/worker.py
View File

@ -512,7 +512,7 @@ class BrozzlerWorker:
url = page.url
if proxy_url:
http = urllib3.ProxyManager("https://%s" % proxy_url, cert_reqs="CERT_NONE")
http = urllib3.ProxyManager("http://%s" % proxy_url, cert_reqs="CERT_NONE")
else:
http = urllib3.PoolManager()
@ -532,11 +532,12 @@ class BrozzlerWorker:
)
except TimeoutError as e:
self.logger.warning("Timed out fetching %s", url)
raise brozzler.ProxyError("proxy error fetching %s" % url) from e
raise brozzler.PageConnectionError() from e
except ProxyError as e:
raise brozzler.ProxyError("proxy error fetching %s" % url) from e
except urllib3.exceptions.RequestError as e:
self.logger.warning("Failed to fetch url %s", page.url, e)
self.logger.warning("Failed to fetch url %s: %s", url, e)
raise brozzler.PageConnectionError() from e
def brozzle_site(self, browser, site):
try: