Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-23 16:19:49 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

add brozzler_ydl_download_successes metric

Browse Source
This commit is contained in:
Barbara Miller 2024-09-19 16:45:22 -07:00
parent 7fbee54eaa
commit 229d53d4b1
2 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
brozzler/metrics.py
View File

@ -29,6 +29,7 @@ brozzler_resources_fetch_time = Counter("brozzler_resources_fetch_time", "time s
brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp")
brozzler_ydl_extract_attempts = Counter("brozzler_ydl_extract_attempts", "count of extracts attempted by brozzler yt-dlp", labelnames=["youtube_host"])
brozzler_ydl_extract_successes = Counter("brozzler_ydl_extract_successes", "count of extracts completed by brozzler yt-dlp", labelnames=["youtube_host"])
brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["youtube_host"])
# fmt: on

19
brozzler/ydl.py
View File

@ -33,7 +33,7 @@ import time
thread_local = threading.local()
PROXYRACK_PROXY = ""
YTDLP_PROXY = ""
MAX_YTDLP_ATTEMPTS = 4
YTDLP_WAIT = 10
@ -239,6 +239,14 @@ def _build_youtube_dl(worker, destdir, site, page):
worker.logger.info(
"[ydl_postprocess_hook] postprocessor: {}".format(d["postprocessor"])
)
youtube_host = (
"youtube.com"
in d["info_dict"]["webpage_url"]
.split("//")[-1]
.split("/")[0]
.split("?")[0]
)
metrics.brozzler_ydl_download_successes.labels(youtube_host).inc(1)
if worker._using_warcprox(site):
_YoutubeDL._push_video_to_warcprox(
_YoutubeDL, site, d["info_dict"], d["postprocessor"]
@ -281,7 +289,8 @@ def _build_youtube_dl(worker, destdir, site, page):
"youtube.com" in ytdlp_url.split("//")[-1].split("/")[0].split("?")[0]
)
if youtube_host:
ydl_opts["proxy"] = PROXYRACK_PROXY
ydl_opts["proxy"] = YTDLP_PROXY
logging.info("using yt-dlp proxy %s", YTDLP_PROXY)
# skip warcprox proxying yt-dlp v.2023.07.06
# if worker._proxy_for(site):
@ -361,7 +370,7 @@ def _try_youtube_dl(worker, ydl, site, page):
"Failed after %s attempts. Error: %s", MAX_YTDLP_ATTEMPTS, e
)
raise brozzler.ProxyError(
"yt-dlp hit possible proxyrack proxy error from %s" % ydl.url
"yt-dlp hit possible external proxy error from %s" % ydl.url
)
else:
logging.info(
@ -371,9 +380,7 @@ def _try_youtube_dl(worker, ydl, site, page):
)
time.sleep(YTDLP_WAIT)
else:
raise brozzler.ProxyError(
"Proxyrack proxy attempt(s) failed for unknown reason(s)"
)
raise brozzler.ProxyError("Proxy attempt(s) failed for unknown reason(s)")
logging.info("ytdlp completed successfully")
_remember_videos(page, ydl.pushed_videos)