diff --git a/brozzler/metrics.py b/brozzler/metrics.py index 0ca3e43..ad5737a 100644 --- a/brozzler/metrics.py +++ b/brozzler/metrics.py @@ -29,6 +29,7 @@ brozzler_resources_fetch_time = Counter("brozzler_resources_fetch_time", "time s brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp") brozzler_ydl_extract_attempts = Counter("brozzler_ydl_extract_attempts", "count of extracts attempted by brozzler yt-dlp", labelnames=["youtube_host"]) brozzler_ydl_extract_successes = Counter("brozzler_ydl_extract_successes", "count of extracts completed by brozzler yt-dlp", labelnames=["youtube_host"]) +brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["youtube_host"]) # fmt: on diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 63c8670..747fa64 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -33,7 +33,7 @@ import time thread_local = threading.local() -PROXYRACK_PROXY = "" +YTDLP_PROXY = "" MAX_YTDLP_ATTEMPTS = 4 YTDLP_WAIT = 10 @@ -239,6 +239,14 @@ def _build_youtube_dl(worker, destdir, site, page): worker.logger.info( "[ydl_postprocess_hook] postprocessor: {}".format(d["postprocessor"]) ) + youtube_host = ( + "youtube.com" + in d["info_dict"]["webpage_url"] + .split("//")[-1] + .split("/")[0] + .split("?")[0] + ) + metrics.brozzler_ydl_download_successes.labels(youtube_host).inc(1) if worker._using_warcprox(site): _YoutubeDL._push_video_to_warcprox( _YoutubeDL, site, d["info_dict"], d["postprocessor"] @@ -281,7 +289,8 @@ def _build_youtube_dl(worker, destdir, site, page): "youtube.com" in ytdlp_url.split("//")[-1].split("/")[0].split("?")[0] ) if youtube_host: - ydl_opts["proxy"] = PROXYRACK_PROXY + ydl_opts["proxy"] = YTDLP_PROXY + logging.info("using yt-dlp proxy %s", YTDLP_PROXY) # skip warcprox proxying yt-dlp v.2023.07.06 # if worker._proxy_for(site): @@ -361,7 +370,7 @@ def _try_youtube_dl(worker, ydl, site, page): "Failed after %s attempts. Error: %s", MAX_YTDLP_ATTEMPTS, e ) raise brozzler.ProxyError( - "yt-dlp hit possible proxyrack proxy error from %s" % ydl.url + "yt-dlp hit possible external proxy error from %s" % ydl.url ) else: logging.info( @@ -371,9 +380,7 @@ def _try_youtube_dl(worker, ydl, site, page): ) time.sleep(YTDLP_WAIT) else: - raise brozzler.ProxyError( - "Proxyrack proxy attempt(s) failed for unknown reason(s)" - ) + raise brozzler.ProxyError("Proxy attempt(s) failed for unknown reason(s)") logging.info("ytdlp completed successfully") _remember_videos(page, ydl.pushed_videos)