Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-23 16:19:49 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

define MAX_YTDLP_ATTEMPTS at head of file

Browse Source
This commit is contained in:
Barbara Miller 2024-12-09 18:45:25 -08:00
parent 1af418c192
commit 36d6a1e5ab
1 changed files with 12 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
brozzler/ydl.py
View File

@ -34,10 +34,10 @@ import time
thread_local = threading.local() thread_local = threading.local()
ytdlp_proxy = "" YTDLP_PROXY = ""
ytdlp_tmp = "/tmp" YTDLP_TMP = "/tmp"
ytdlp_wait = 10 YTDLP_WAIT = 10
max_ytdlp_attempts = 1 MAX_YTDLP_ATTEMPTS = 4 if YTDLP_PROXY else 1
def should_ytdlp(site, page, page_status, skip_av_seeds): def should_ytdlp(site, page, page_status, skip_av_seeds):
@ -286,12 +286,11 @@ def _build_youtube_dl(worker, destdir, site, page):
ytdlp_url = page.redirect_url if page.redirect_url else page.url ytdlp_url = page.redirect_url if page.redirect_url else page.url
is_youtube_host = isyoutubehost(ytdlp_url) is_youtube_host = isyoutubehost(ytdlp_url)
if is_youtube_host and ytdlp_proxy: if is_youtube_host and YTDLP_PROXY:
max_ytdlp_attempts = 4 ydl_opts["proxy"] = YTDLP_PROXY
ydl_opts["proxy"] = ytdlp_proxy
# don't log proxy value secrets # don't log proxy value secrets
ytdlp_proxy_for_logs = ( ytdlp_proxy_for_logs = (
ytdlp_proxy.split("@")[1] if "@" in ytdlp_proxy else "@@@" YTDLP_PROXY.split("@")[1] if "@" in YTDLP_PROXY else "@@@"
) )
logging.info("using yt-dlp proxy ... %s", ytdlp_proxy_for_logs) logging.info("using yt-dlp proxy ... %s", ytdlp_proxy_for_logs)
@ -329,7 +328,7 @@ def _remember_videos(page, pushed_videos=None):
def _try_youtube_dl(worker, ydl, site, page): def _try_youtube_dl(worker, ydl, site, page):
attempt = 0 attempt = 0
while attempt < max_ytdlp_attempts: while attempt < MAX_YTDLP_ATTEMPTS:
try: try:
logging.info("trying yt-dlp on %s", ydl.url) logging.info("trying yt-dlp on %s", ydl.url)
# should_download_vid = not ydl.is_youtube_host # should_download_vid = not ydl.is_youtube_host
@ -367,15 +366,15 @@ def _try_youtube_dl(worker, ydl, site, page):
# OSError('Tunnel connection failed: 464 Host Not Allowed') (caused by ProxyError...) # OSError('Tunnel connection failed: 464 Host Not Allowed') (caused by ProxyError...)
# and others... # and others...
attempt += 1 attempt += 1
if attempt == max_ytdlp_attempts: if attempt == MAX_YTDLP_ATTEMPTS:
logging.warning( logging.warning(
"Failed after %s attempts. Error: %s", max_ytdlp_attempts, e "Failed after %s attempts. Error: %s", MAX_YTDLP_ATTEMPTS, e
) )
raise brozzler.VideoExtractorError( raise brozzler.VideoExtractorError(
"yt-dlp hit error extracting info for %s" % ydl.url "yt-dlp hit error extracting info for %s" % ydl.url
) )
else: else:
retry_wait = min(60, ytdlp_wait * (1.5 ** (attempt - 1))) retry_wait = min(60, YTDLP_WAIT * (1.5 ** (attempt - 1)))
logging.info( logging.info(
"Attempt %s failed. Retrying in %s seconds...", "Attempt %s failed. Retrying in %s seconds...",
attempt, attempt,
@ -423,7 +422,7 @@ def do_youtube_dl(worker, site, page):
Returns: Returns:
`list` of `str`: outlink urls `list` of `str`: outlink urls
""" """
with tempfile.TemporaryDirectory(prefix="brzl-ydl-") as tempdir: with tempfile.TemporaryDirectory(prefix="brzl-ydl-", dir=YTDLP_TMP) as tempdir:
ydl = _build_youtube_dl(worker, tempdir, site, page) ydl = _build_youtube_dl(worker, tempdir, site, page)
ie_result = _try_youtube_dl(worker, ydl, site, page) ie_result = _try_youtube_dl(worker, ydl, site, page)
outlinks = set() outlinks = set()