Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-25 00:59:52 -05:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

tidier get_page_headers

Browse Source
This commit is contained in:
Barbara Miller 2024-10-08 16:35:34 -07:00
parent 151777298a
commit 90dbdbd19d
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
brozzler/worker.py
View File

@ -318,13 +318,13 @@ class BrozzlerWorker:
def _get_page_headers(self, page): def _get_page_headers(self, page):
# bypassing warcprox, requests' stream=True defers downloading the body of the response # bypassing warcprox, requests' stream=True defers downloading the body of the response
# see https://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow # see https://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow
page_headers = {}
try: try:
with requests.get(page.url, stream=True, verify=False) as r: with requests.get(page.url, stream=True, verify=False) as r:
page_headers = r.headers page_headers = r.headers
return page_headers
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
self.logger.warning("Failed to get headers for %s: %s", page.url, e) self.logger.warning("Failed to get headers for %s: %s", page.url, e)
return {} return page_headers
def _needs_browsing(self, page_headers): def _needs_browsing(self, page_headers):
if ( if (