Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 16:49:56 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'metrics_plus_proxy_retries' into qa

Browse Source
This commit is contained in:
Barbara Miller 2024-09-19 10:54:55 -07:00
commit 3b4db5c74d
2 changed files with 25 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
brozzler/ydl.py
View File

@ -38,7 +38,7 @@ import time
thread_local = threading.local() thread_local = threading.local()
PROXYRACK_PROXY = "@@@" PROXYRACK_PROXY = ""
MAX_YTDLP_ATTEMPTS = 4 MAX_YTDLP_ATTEMPTS = 4
YTDLP_WAIT = 10 YTDLP_WAIT = 10
@ -318,13 +318,19 @@ def _build_youtube_dl(worker, destdir, site, page):
"logger": logging.getLogger("yt_dlp"), "logger": logging.getLogger("yt_dlp"),
"verbose": False, "verbose": False,
"quiet": False, "quiet": False,
# does this make sense when we're generally downloading one at a time? # recommended to avoid bot detection
"sleep_interval": 25, "sleep_interval": 25,
"max_sleep_interval": 90, "max_sleep_interval": 90,
"proxy": PROXYRACK_PROXY,
} }
# skip proxying yt-dlp v.2023.07.06 ytdlp_url = page.redirect_url if page.redirect_url else page.url
youtube_host = (
"youtube.com" in ytdlp_url.split("//")[-1].split("/")[0].split("?")[0]
)
if youtube_host:
ydl_opts["proxy"] = PROXYRACK_PROXY
# skip warcprox proxying yt-dlp v.2023.07.06
# if worker._proxy_for(site): # if worker._proxy_for(site):
# ydl_opts["proxy"] = "http://{}".format(worker._proxy_for(site)) # ydl_opts["proxy"] = "http://{}".format(worker._proxy_for(site))
@ -332,6 +338,9 @@ def _build_youtube_dl(worker, destdir, site, page):
if site.extra_headers(): if site.extra_headers():
ydl._opener.add_handler(ExtraHeaderAdder(site.extra_headers(page))) ydl._opener.add_handler(ExtraHeaderAdder(site.extra_headers(page)))
ydl.pushed_videos = [] ydl.pushed_videos = []
ydl.url = ytdlp_url
ydl.youtube_host = youtube_host
return ydl return ydl
@ -354,29 +363,25 @@ def _remember_videos(page, pushed_videos=None):
def _try_youtube_dl(worker, ydl, site, page): def _try_youtube_dl(worker, ydl, site, page):
ytdlp_url = page.redirect_url if page.redirect_url else page.url
youtube_host = (
"youtube.com" in ytdlp_url.split("//")[-1].split("/")[0].split("?")[0]
)
attempt = 0 attempt = 0
while attempt < MAX_YTDLP_ATTEMPTS: while attempt < MAX_YTDLP_ATTEMPTS:
try: try:
logging.info("trying yt-dlp on %s", ytdlp_url) logging.info("trying yt-dlp on %s", ydl.url)
# should_download_vid = not youtube_host # should_download_vid = not ydl.youtube_host
# then # then
# ydl.extract_info(str(urlcanon.whatwg(ytdlp_url)), download=should_download_vid) # ydl.extract_info(str(urlcanon.whatwg(ydl.url)), download=should_download_vid)
# if youtube_host and ie_result: # if ydl.youtube_host and ie_result:
# download_url = ie_result.get("url") # download_url = ie_result.get("url")
metrics.brozzler_ydl_extract_attempts.labels(youtube_host).inc(1) metrics.brozzler_ydl_extract_attempts.labels(ydl.youtube_host).inc(1)
with brozzler.thread_accept_exceptions(): with brozzler.thread_accept_exceptions():
# we do whatwg canonicalization here to avoid "<urlopen error # we do whatwg canonicalization here to avoid "<urlopen error
# no host given>" resulting in ProxyError # no host given>" resulting in ProxyError
# needs automated test # needs automated test
# and yt-dlp needs sanitize_info for extract_info # and yt-dlp needs sanitize_info for extract_info
ie_result = ydl.sanitize_info( ie_result = ydl.sanitize_info(
ydl.extract_info(str(urlcanon.whatwg(ytdlp_url))) ydl.extract_info(str(urlcanon.whatwg(ydl.url)))
) )
metrics.brozzler_ydl_extract_successes.labels(youtube_host).inc(1) metrics.brozzler_ydl_extract_successes.labels(ydl.youtube_host).inc(1)
break break
except brozzler.ShutdownRequested as e: except brozzler.ShutdownRequested as e:
raise raise
@ -394,6 +399,7 @@ def _try_youtube_dl(worker, ydl, site, page):
): ):
raise brozzler.ReachedLimit(e.exc_info[1]) raise brozzler.ReachedLimit(e.exc_info[1])
else: else:
# todo: other errors to handle separately?
# OSError('Tunnel connection failed: 464 Host Not Allowed') (caused by ProxyError...) # OSError('Tunnel connection failed: 464 Host Not Allowed') (caused by ProxyError...)
# and others... # and others...
attempt += 1 attempt += 1
@ -402,7 +408,7 @@ def _try_youtube_dl(worker, ydl, site, page):
"Failed after %s attempts. Error: %s", MAX_YTDLP_ATTEMPTS, e "Failed after %s attempts. Error: %s", MAX_YTDLP_ATTEMPTS, e
) )
raise brozzler.ProxyError( raise brozzler.ProxyError(
"yt-dlp hit proxyrack proxy error from %s" % ytdlp_url "yt-dlp hit possible proxyrack proxy error from %s" % ydl.url
) )
else: else:
logging.info( logging.info(
@ -423,11 +429,11 @@ def _try_youtube_dl(worker, ydl, site, page):
logging.info( logging.info(
"sending WARCPROX_WRITE_RECORD request to warcprox " "sending WARCPROX_WRITE_RECORD request to warcprox "
"with yt-dlp json for %s", "with yt-dlp json for %s",
ytdlp_url, ydl.url,
) )
worker._warcprox_write_record( worker._warcprox_write_record(
warcprox_address=worker._proxy_for(site), warcprox_address=worker._proxy_for(site),
url="youtube-dl:%s" % str(urlcanon.semantic(ytdlp_url)), url="youtube-dl:%s" % str(urlcanon.semantic(ydl.url)),
warc_type="metadata", warc_type="metadata",
content_type="application/vnd.youtube-dl_formats+json;charset=utf-8", content_type="application/vnd.youtube-dl_formats+json;charset=utf-8",
payload=info_json.encode("utf-8"), payload=info_json.encode("utf-8"),

2
setup.py
View File

@ -34,7 +34,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name="brozzler", name="brozzler",
version="1.5.55a2", version="1.5.55a3",
description="Distributed web crawling with browsers", description="Distributed web crawling with browsers",
url="https://github.com/internetarchive/brozzler", url="https://github.com/internetarchive/brozzler",
author="Noah Levitt", author="Noah Levitt",