Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-19 23:35:54 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #284 from internetarchive/adam/adds_get_page_header_exception_handling

Browse Source 
fix: handle exceptions when requesting page headers for content-type …
This commit is contained in:
Adam Miller 2024-08-19 15:31:13 -07:00 committed by GitHub
commit 68a6953077
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
brozzler/worker.py
View File

@ -295,9 +295,13 @@ class BrozzlerWorker:
def _get_page_headers(self, page):
# bypassing warcprox, requests' stream=True defers downloading the body of the response
# see https://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow
with requests.get(page.url, stream=True) as r:
page_headers = r.headers
return page_headers
try:
with requests.get(page.url, stream=True, verify=False) as r:
page_headers = r.headers
return page_headers
except requests.exceptions.RequestException as e:
self.logger.warning("Failed to get headers for %s: %s", page.url, e)
return {}
def _needs_browsing(self, page_headers):
if (