Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-22 04:39:36 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

if not self._needs_browsing; _get_page_headers comment

Browse source
This commit is contained in:
Barbara Miller 2024-04-23 16:01:35 -07:00
parent 7764c3f6d7
commit 2d183c7d0c
1 changed files with 6 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

10
brozzler/worker.py
View file

@ -246,7 +246,10 @@ class BrozzlerWorker:
self._get_page_headers(page) self._get_page_headers(page)
if self._needs_browsing(page): if not self._needs_browsing(page):
self.logger.info("needs fetch: %s", page)
self._fetch_url(site, page=page)
else:
self.logger.info("needs browsing: %s", page) self.logger.info("needs browsing: %s", page)
try: try:
browser_outlinks = self._browse_page( browser_outlinks = self._browse_page(
@ -285,13 +288,12 @@ class BrozzlerWorker:
self.logger.error( self.logger.error(
"youtube_dl raised exception on %s", page, exc_info=True "youtube_dl raised exception on %s", page, exc_info=True
) )
else:
self.logger.info("needs fetch: %s", page)
self._fetch_url(site, page=page)
return outlinks return outlinks
def _get_page_headers(self, page): def _get_page_headers(self, page):
page.content_type = page.content_length = page.last_modified = None page.content_type = page.content_length = page.last_modified = None
# bypassing warcprox, requests' stream=True defers downloading the body of the response
# see https://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow
with requests.get(page.url, stream=True) as r: with requests.get(page.url, stream=True) as r:
if "content-type" in r.headers: if "content-type" in r.headers:
page.content_type = r.headers["content-type"] page.content_type = r.headers["content-type"]