diff --git a/brozzler/worker.py b/brozzler/worker.py index 86977cf..2bad677 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -3,7 +3,7 @@ brozzler/worker.py - BrozzlerWorker brozzles pages from the frontier, meaning it runs yt-dlp on them, browses them and runs behaviors if appropriate, scopes and adds outlinks to the frontier -Copyright (C) 2014-2023 Internet Archive +Copyright (C) 2014-2024 Internet Archive Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -242,53 +242,82 @@ class BrozzlerWorker: enable_youtube_dl=True, ): self.logger.info("brozzling {}".format(page)) - ydl_fetches = None outlinks = set() - if enable_youtube_dl and not page.url.lower().endswith(".pdf"): - try: - ydl_fetches, outlinks = ydl.do_youtube_dl(self, site, page) - except brozzler.ReachedLimit as e: - raise - except brozzler.ShutdownRequested: - raise - except brozzler.ProxyError: - raise - except Exception as e: - if ( - hasattr(e, "exc_info") - and len(e.exc_info) >= 2 - and hasattr(e.exc_info[1], "code") - and e.exc_info[1].code == 430 - ): - self.logger.info( - "youtube-dl got %s %s processing %s", - e.exc_info[1].code, - e.exc_info[1].msg, - page.url, - ) - else: - self.logger.error( - "youtube_dl raised exception on %s", page, exc_info=True - ) - if self._needs_browsing(page, ydl_fetches): + self._get_page_headers(page) + + if not self._needs_browsing(page): + self.logger.info("needs fetch: %s", page) + self._fetch_url(site, page=page) + else: self.logger.info("needs browsing: %s", page) try: browser_outlinks = self._browse_page( browser, site, page, on_screenshot, on_request ) outlinks.update(browser_outlinks) + page.status_code = browser.websock_thread.page_status + self.logger.info("url %s status code %s", page.url, page.status_code) except brozzler.PageInterstitialShown: self.logger.info("page interstitial shown (http auth): %s", page) - else: - if not self._already_fetched(page, ydl_fetches): - self.logger.info("needs fetch: %s", page) - self._fetch_url(site, page=page) - else: - self.logger.info("already fetched: %s", page) + if enable_youtube_dl and ydl.should_ytdlp(page, site): + try: + ydl_outlinks = ydl.do_youtube_dl(self, site, page) + outlinks.update(ydl_outlinks) + except brozzler.ReachedLimit as e: + raise + except brozzler.ShutdownRequested: + raise + except brozzler.ProxyError: + raise + except Exception as e: + if ( + hasattr(e, "exc_info") + and len(e.exc_info) >= 2 + and hasattr(e.exc_info[1], "code") + and e.exc_info[1].code == 430 + ): + self.logger.info( + "youtube-dl got %s %s processing %s", + e.exc_info[1].code, + e.exc_info[1].msg, + page.url, + ) + else: + self.logger.error( + "youtube_dl raised exception on %s", page, exc_info=True + ) return outlinks + def _get_page_headers(self, page): + page.content_type = page.content_length = page.last_modified = None + # bypassing warcprox, requests' stream=True defers downloading the body of the response + # see https://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow + with requests.get(page.url, stream=True) as r: + if "content-type" in r.headers: + page.content_type = r.headers["content-type"] + self.logger.info( + "content_type: %s for url %s", page.content_type, page.url + ) + + if "content-length" in r.headers: + page.content_length = int(r.headers["content-length"]) + self.logger.info( + "content_length: %s for url %s", page.content_length, page.url + ) + + if "last-modified" in r.headers: + page.last_modified = r.headers["last-modified"] + self.logger.info( + "last_modified: %s for url %s", page.last_modified, page.url + ) + + def _needs_browsing(self, page): + if page.content_type and "html" not in page.content_type: + return False + return True + def _browse_page(self, browser, site, page, on_screenshot=None, on_request=None): def _on_screenshot(screenshot_jpeg): if on_screenshot: @@ -415,28 +444,6 @@ class BrozzlerWorker: except requests.exceptions.ProxyError as e: raise brozzler.ProxyError("proxy error fetching %s" % url) from e - def _needs_browsing(self, page, ydl_fetches): - if ydl_fetches: - final_bounces = ydl.final_bounces(ydl_fetches, page.url) - if not final_bounces: - return True - for txn in final_bounces: - if txn["response_headers"].get_content_type() in [ - "text/html", - "application/xhtml+xml", - ]: - return True - return False - else: - return True - - def _already_fetched(self, page, ydl_fetches): - if ydl_fetches: - for fetch in ydl.final_bounces(ydl_fetches, page.url): - if fetch["method"] == "GET" and fetch["response_code"] == 200: - return True - return False - def brozzle_site(self, browser, site): try: site.last_claimed_by = "%s:%s" % (socket.gethostname(), browser.chrome.port) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 4281d4a..9caf662 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -1,7 +1,7 @@ """ brozzler/ydl.py - youtube-dl / yt-dlp support for brozzler -Copyright (C) 2023 Internet Archive +Copyright (C) 2024 Internet Archive Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -32,6 +32,19 @@ import threading thread_local = threading.local() +def should_ytdlp(page, site): + # called only after we've passed needs_browsing() check + if page.status_code != 200: + return False + + ytdlp_url = page.redirect_url if page.redirect_url else page.url + + if "chrome-error:" in ytdlp_url: + return False + + return True + + class ExtraHeaderAdder(urllib.request.BaseHandler): def __init__(self, extra_headers): self.extra_headers = extra_headers @@ -45,64 +58,12 @@ class ExtraHeaderAdder(urllib.request.BaseHandler): return req -class YoutubeDLSpy(urllib.request.BaseHandler): - logger = logging.getLogger(__module__ + "." + __qualname__) - - def __init__(self): - self.reset() - - def _http_response(self, request, response): - fetch = { - "url": request.full_url, - "method": request.get_method(), - "response_code": response.code, - "response_headers": response.headers, - } - self.fetches.append(fetch) - return response - - http_response = https_response = _http_response - - def reset(self): - self.fetches = [] - - -def final_bounces(fetches, url): - """ - Resolves redirect chains in `fetches` and returns a list of fetches - representing the final redirect destinations of the given url. There could - be more than one if for example youtube-dl hit the same url with HEAD and - then GET requests. - """ - redirects = {} - for fetch in fetches: - # XXX check http status 301,302,303,307? check for "uri" header - # as well as "location"? see urllib.request.HTTPRedirectHandler - if "location" in fetch["response_headers"]: - redirects[fetch["url"]] = fetch - - final_url = url - while final_url in redirects: - fetch = redirects.pop(final_url) - final_url = urllib.parse.urljoin( - fetch["url"], fetch["response_headers"]["location"] - ) - - final_bounces = [] - for fetch in fetches: - if fetch["url"] == final_url: - final_bounces.append(fetch) - - return final_bounces - - def _build_youtube_dl(worker, destdir, site, page): """ Builds a yt-dlp `yt_dlp.YoutubeDL` for brozzling `site` with `worker`. The `YoutubeDL` instance does a few special brozzler-specific things: - - keeps track of urls fetched using a `YoutubeDLSpy` - periodically updates `site.last_claimed` in rethinkdb - pushes captured video to warcprox using a WARCPROX_WRITE_RECORD request - some logging @@ -111,6 +72,7 @@ def _build_youtube_dl(worker, destdir, site, page): worker (brozzler.BrozzlerWorker): the calling brozzler worker destdir (str): where to save downloaded videos site (brozzler.Site): the site we are brozzling + page (brozzler.Page): the page we are brozzling Returns: a yt-dlp `yt_dlp.YoutubeDL` instance @@ -183,8 +145,8 @@ def _build_youtube_dl(worker, destdir, site, page): else: url = info_dict.get("url", "") - # skip urls ending .m3u8, to avoid duplicates handled by FixupM3u8 - if url.endswith(".m3u8") or url == "": + # skip urls containing .m3u8, to avoid duplicates handled by FixupM3u8 + if url == "" or ".m3u8" in url: return size = os.path.getsize(info_dict["filepath"]) @@ -277,7 +239,7 @@ def _build_youtube_dl(worker, destdir, site, page): "match_filter": match_filter_func("!is_live"), "extractor_args": {"youtube": {"skip": ["dash", "hls"]}}, # --cache-dir local or.. - # this looked like a problem with nsf-mounted homedir, shouldn't be a problem for brozzler on focal? + # this looked like a problem with nsf-mounted homedir, maybe not a problem for brozzler on focal? "cache_dir": "/home/archiveit", "logger": logging.getLogger("yt_dlp"), "verbose": False, @@ -291,61 +253,33 @@ def _build_youtube_dl(worker, destdir, site, page): ydl = _YoutubeDL(ydl_opts) if site.extra_headers(): ydl._opener.add_handler(ExtraHeaderAdder(site.extra_headers(page))) - ydl.fetch_spy = YoutubeDLSpy() ydl.pushed_videos = [] - ydl._opener.add_handler(ydl.fetch_spy) + return ydl -def _remember_videos(page, fetches, pushed_videos=None): +def _remember_videos(page, pushed_videos=None): """ Saves info about videos captured by yt-dlp in `page.videos`. """ if not "videos" in page: page.videos = [] - for fetch in fetches or []: - content_type = fetch["response_headers"].get_content_type() - if ( - content_type.startswith("video/") - # skip manifests of DASH segmented video - - # see https://github.com/internetarchive/brozzler/pull/70 - and content_type != "video/vnd.mpeg.dash.mpd" - and fetch["method"] == "GET" - and fetch["response_code"] in (200, 206) - ): - video = { - "blame": "youtube-dl", - "url": fetch["url"], - "response_code": fetch["response_code"], - "content-type": content_type, - } - if "content-length" in fetch["response_headers"]: - video["content-length"] = int( - fetch["response_headers"]["content-length"] - ) - if "content-range" in fetch["response_headers"]: - # skip chunked youtube video - if "googlevideo.com/videoplayback" in fetch["url"]: - continue - video["content-range"] = fetch["response_headers"]["content-range"] - logging.debug("embedded video %s", video) - page.videos.append(video) for pushed_video in pushed_videos or []: - if pushed_video["content-type"].startswith("video/"): - video = { - "blame": "youtube-dl", - "url": pushed_video["url"], - "response_code": pushed_video["response_code"], - "content-type": pushed_video["content-type"], - "content-length": pushed_video["content-length"], - } - logging.debug("embedded video %s", video) - page.videos.append(video) + video = { + "blame": "youtube-dl", + "url": pushed_video["url"], + "response_code": pushed_video["response_code"], + "content-type": pushed_video["content-type"], + "content-length": pushed_video["content-length"], + } + logging.debug("embedded video %s", video) + page.videos.append(video) def _try_youtube_dl(worker, ydl, site, page): + ytdlp_url = page.redirect_url if page.redirect_url else page.url try: - logging.info("trying yt-dlp on %s", page) + logging.info("trying yt-dlp on %s", ytdlp_url) with brozzler.thread_accept_exceptions(): # we do whatwg canonicalization here to avoid "