Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-07-26 08:15:43 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'metrics_plus_proxy_retries' into qa

Browse source
This commit is contained in:
Barbara Miller 2024-09-19 19:48:24 -07:00
commit cb4da54e36
2 changed files with 14 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

1
brozzler/metrics.py
View file

@ -29,6 +29,7 @@ brozzler_resources_fetch_time = Counter("brozzler_resources_fetch_time", "time s
brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp") brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp")
brozzler_ydl_extract_attempts = Counter("brozzler_ydl_extract_attempts", "count of extracts attempted by brozzler yt-dlp", labelnames=["youtube_host"]) brozzler_ydl_extract_attempts = Counter("brozzler_ydl_extract_attempts", "count of extracts attempted by brozzler yt-dlp", labelnames=["youtube_host"])
brozzler_ydl_extract_successes = Counter("brozzler_ydl_extract_successes", "count of extracts completed by brozzler yt-dlp", labelnames=["youtube_host"]) brozzler_ydl_extract_successes = Counter("brozzler_ydl_extract_successes", "count of extracts completed by brozzler yt-dlp", labelnames=["youtube_host"])
brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["youtube_host"])
# fmt: on # fmt: on

19
brozzler/ydl.py
View file

@ -38,7 +38,7 @@ import time
thread_local = threading.local() thread_local = threading.local()
PROXYRACK_PROXY = "" YTDLP_PROXY = ""
MAX_YTDLP_ATTEMPTS = 4 MAX_YTDLP_ATTEMPTS = 4
YTDLP_WAIT = 10 YTDLP_WAIT = 10
@ -286,6 +286,14 @@ def _build_youtube_dl(worker, destdir, site, page):
worker.logger.info( worker.logger.info(
"[ydl_postprocess_hook] postprocessor: {}".format(d["postprocessor"]) "[ydl_postprocess_hook] postprocessor: {}".format(d["postprocessor"])
) )
youtube_host = (
"youtube.com"
in d["info_dict"]["webpage_url"]
.split("//")[-1]
.split("/")[0]
.split("?")[0]
)
metrics.brozzler_ydl_download_successes.labels(youtube_host).inc(1)
if worker._using_warcprox(site): if worker._using_warcprox(site):
_YoutubeDL._push_video_to_warcprox( _YoutubeDL._push_video_to_warcprox(
_YoutubeDL, site, d["info_dict"], d["postprocessor"] _YoutubeDL, site, d["info_dict"], d["postprocessor"]
@ -328,7 +336,8 @@ def _build_youtube_dl(worker, destdir, site, page):
"youtube.com" in ytdlp_url.split("//")[-1].split("/")[0].split("?")[0] "youtube.com" in ytdlp_url.split("//")[-1].split("/")[0].split("?")[0]
) )
if youtube_host: if youtube_host:
ydl_opts["proxy"] = PROXYRACK_PROXY ydl_opts["proxy"] = YTDLP_PROXY
logging.info("using yt-dlp proxy %s", YTDLP_PROXY)
# skip warcprox proxying yt-dlp v.2023.07.06 # skip warcprox proxying yt-dlp v.2023.07.06
# if worker._proxy_for(site): # if worker._proxy_for(site):
@ -408,7 +417,7 @@ def _try_youtube_dl(worker, ydl, site, page):
"Failed after %s attempts. Error: %s", MAX_YTDLP_ATTEMPTS, e "Failed after %s attempts. Error: %s", MAX_YTDLP_ATTEMPTS, e
) )
raise brozzler.ProxyError( raise brozzler.ProxyError(
"yt-dlp hit possible proxyrack proxy error from %s" % ydl.url "yt-dlp hit possible external proxy error from %s" % ydl.url
) )
else: else:
logging.info( logging.info(
@ -418,9 +427,7 @@ def _try_youtube_dl(worker, ydl, site, page):
) )
time.sleep(YTDLP_WAIT) time.sleep(YTDLP_WAIT)
else: else:
raise brozzler.ProxyError( raise brozzler.ProxyError("Proxy attempt(s) failed for unknown reason(s)")
"Proxyrack proxy attempt(s) failed for unknown reason(s)"
)
logging.info("ytdlp completed successfully") logging.info("ytdlp completed successfully")
_remember_videos(page, ydl.pushed_videos) _remember_videos(page, ydl.pushed_videos)