Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-23 16:19:49 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

mostly ydl.py updates for new proxyrack testing

Browse Source
This commit is contained in:
Barbara Miller 2024-09-18 14:51:11 -07:00
parent 8b2c254485
commit 62b12434d0
1 changed files with 28 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
brozzler/ydl.py
View File

@ -34,7 +34,7 @@ import time
thread_local = threading.local() thread_local = threading.local()
PROXYRACK_PROXY = "@@@" PROXYRACK_PROXY = "@@@"
MAX_YTDLP_ATTEMPTS = 4 MAX_YTDLP_ATTEMPTS = 3
YTDLP_WAIT = 10 YTDLP_WAIT = 10
@ -197,15 +197,15 @@ def _build_youtube_dl(worker, destdir, site, page):
payload=f, payload=f,
extra_headers=extra_headers, extra_headers=extra_headers,
) )
# consulted by _remember_videos() # consulted by _remember_videos()
ydl.pushed_videos.append( ydl.pushed_videos.append(
{ {
"url": url, "url": url,
"response_code": response.code, "response_code": response.code,
"content-type": mimetype, "content-type": mimetype,
"content-length": size, "content-length": size,
} }
) )
def maybe_heartbeat_site_last_claimed(*args, **kwargs): def maybe_heartbeat_site_last_claimed(*args, **kwargs):
# in case yt-dlp takes a long time, heartbeat site.last_claimed # in case yt-dlp takes a long time, heartbeat site.last_claimed
@ -268,6 +268,9 @@ def _build_youtube_dl(worker, destdir, site, page):
"logger": logging.getLogger("yt_dlp"), "logger": logging.getLogger("yt_dlp"),
"verbose": False, "verbose": False,
"quiet": False, "quiet": False,
# does this make sense when we're generally downloading one at a time?
"sleep_interval": 25,
"max_sleep_interval": 90,
"proxy": PROXYRACK_PROXY, "proxy": PROXYRACK_PROXY,
} }
@ -308,7 +311,12 @@ def _try_youtube_dl(worker, ydl, site, page):
while attempt < MAX_YTDLP_ATTEMPTS: while attempt < MAX_YTDLP_ATTEMPTS:
try: try:
logging.info("trying yt-dlp on %s", ytdlp_url) logging.info("trying yt-dlp on %s", ytdlp_url)
metrics.brozzler_ydl_download_attempts.labels(youtube_host).inc(1) # should_download_vid = not youtube_host
# then
# ydl.extract_info(str(urlcanon.whatwg(ytdlp_url)), download=should_download_vid)
# if youtube_host and ie_result:
# download_url = ie_result.get("url")
metrics.brozzler_ydl_extract_attempts.labels(youtube_host).inc(1)
with brozzler.thread_accept_exceptions(): with brozzler.thread_accept_exceptions():
# we do whatwg canonicalization here to avoid "<urlopen error # we do whatwg canonicalization here to avoid "<urlopen error
# no host given>" resulting in ProxyError # no host given>" resulting in ProxyError
@ -317,7 +325,7 @@ def _try_youtube_dl(worker, ydl, site, page):
ie_result = ydl.sanitize_info( ie_result = ydl.sanitize_info(
ydl.extract_info(str(urlcanon.whatwg(ytdlp_url))) ydl.extract_info(str(urlcanon.whatwg(ytdlp_url)))
) )
metrics.brozzler_ydl_download_successes.labels(youtube_host).inc(1) metrics.brozzler_ydl_extract_successes.labels(youtube_host).inc(1)
break break
except brozzler.ShutdownRequested as e: except brozzler.ShutdownRequested as e:
raise raise
@ -366,48 +374,14 @@ def _try_youtube_dl(worker, ydl, site, page):
"with yt-dlp json for %s", "with yt-dlp json for %s",
ytdlp_url, ytdlp_url,
) )
worker._warcprox_write_record(
attempt = 0 warcprox_address=worker._proxy_for(site),
while attempt < MAX_YTDLP_ATTEMPTS: url="youtube-dl:%s" % str(urlcanon.semantic(ytdlp_url)),
try: warc_type="metadata",
worker._warcprox_write_record( content_type="application/vnd.youtube-dl_formats+json;charset=utf-8",
warcprox_address=worker._proxy_for(site), payload=info_json.encode("utf-8"),
url="youtube-dl:%s" % str(urlcanon.semantic(ytdlp_url)), extra_headers=site.extra_headers(page),
warc_type="metadata", )
content_type="application/vnd.youtube-dl_formats+json;charset=utf-8",
payload=info_json.encode("utf-8"),
extra_headers=site.extra_headers(page),
)
break
except Exception as e:
# connection problem when using a proxy == proxy error
if (
hasattr(e, "exc_info")
and e.exc_info[0] == urllib.error.URLError
and worker._proxy_for(site)
):
attempt += 1
if attempt == MAX_YTDLP_ATTEMPTS:
logging.warning(
"Failed after %s attempts. Error: %s", MAX_YTDLP_ATTEMPTS, e
)
raise brozzler.ProxyError(
"yt-dlp hit proxy error storing media from %s with "
% ytdlp_url
)
else:
logging.info(
"Attempt %s failed. Retrying in %s seconds...",
attempt,
YTDLP_WAIT,
)
time.sleep(YTDLP_WAIT)
else:
raise
else:
raise brozzler.ProxyError(
"Proxy attempt(s) storing media failed for unknown reason(s)"
)
return ie_result return ie_result