diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 38dfcd3..211676b 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -34,10 +34,10 @@ import time thread_local = threading.local() -ytdlp_proxy = "" -ytdlp_tmp = "/tmp" -ytdlp_wait = 10 -max_ytdlp_attempts = 1 +YTDLP_PROXY = "" +YTDLP_TMP = "/tmp" +YTDLP_WAIT = 10 +MAX_YTDLP_ATTEMPTS = 4 if YTDLP_PROXY else 1 def should_ytdlp(site, page, page_status, skip_av_seeds): @@ -286,12 +286,11 @@ def _build_youtube_dl(worker, destdir, site, page): ytdlp_url = page.redirect_url if page.redirect_url else page.url is_youtube_host = isyoutubehost(ytdlp_url) - if is_youtube_host and ytdlp_proxy: - max_ytdlp_attempts = 4 - ydl_opts["proxy"] = ytdlp_proxy + if is_youtube_host and YTDLP_PROXY: + ydl_opts["proxy"] = YTDLP_PROXY # don't log proxy value secrets ytdlp_proxy_for_logs = ( - ytdlp_proxy.split("@")[1] if "@" in ytdlp_proxy else "@@@" + YTDLP_PROXY.split("@")[1] if "@" in YTDLP_PROXY else "@@@" ) logging.info("using yt-dlp proxy ... %s", ytdlp_proxy_for_logs) @@ -329,7 +328,7 @@ def _remember_videos(page, pushed_videos=None): def _try_youtube_dl(worker, ydl, site, page): attempt = 0 - while attempt < max_ytdlp_attempts: + while attempt < MAX_YTDLP_ATTEMPTS: try: logging.info("trying yt-dlp on %s", ydl.url) # should_download_vid = not ydl.is_youtube_host @@ -367,15 +366,15 @@ def _try_youtube_dl(worker, ydl, site, page): # OSError('Tunnel connection failed: 464 Host Not Allowed') (caused by ProxyError...) # and others... attempt += 1 - if attempt == max_ytdlp_attempts: + if attempt == MAX_YTDLP_ATTEMPTS: logging.warning( - "Failed after %s attempts. Error: %s", max_ytdlp_attempts, e + "Failed after %s attempts. Error: %s", MAX_YTDLP_ATTEMPTS, e ) raise brozzler.VideoExtractorError( "yt-dlp hit error extracting info for %s" % ydl.url ) else: - retry_wait = min(60, ytdlp_wait * (1.5 ** (attempt - 1))) + retry_wait = min(60, YTDLP_WAIT * (1.5 ** (attempt - 1))) logging.info( "Attempt %s failed. Retrying in %s seconds...", attempt, @@ -423,7 +422,7 @@ def do_youtube_dl(worker, site, page): Returns: `list` of `str`: outlink urls """ - with tempfile.TemporaryDirectory(prefix="brzl-ydl-") as tempdir: + with tempfile.TemporaryDirectory(prefix="brzl-ydl-", dir=YTDLP_TMP) as tempdir: ydl = _build_youtube_dl(worker, tempdir, site, page) ie_result = _try_youtube_dl(worker, ydl, site, page) outlinks = set()