Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-06 05:24:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #308 from galgeek/retry_only_proxied

Browse source 
retry only proxied urls
This commit is contained in:
Barbara Miller 2024-12-12 13:59:09 -08:00 • committed by GitHub
commit 135f623fda
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 6 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

10
brozzler/ydl.py
View file

@ -33,8 +33,9 @@ import time
thread_local = threading.local() thread_local = threading.local()
YTDLP_PROXY = "" YTDLP_PROXY = ""
MAX_YTDLP_ATTEMPTS = 4 PROXY_ATTEMPTS = 4
YTDLP_WAIT = 10 YTDLP_WAIT = 10
@ -325,8 +326,9 @@ def _remember_videos(page, pushed_videos=None):
def _try_youtube_dl(worker, ydl, site, page): def _try_youtube_dl(worker, ydl, site, page):
max_attempts = PROXY_ATTEMPTS if ydl.is_youtube_host else 1
attempt = 0 attempt = 0
while attempt < MAX_YTDLP_ATTEMPTS: while attempt < max_attempts:
try: try:
logging.info("trying yt-dlp on %s", ydl.url) logging.info("trying yt-dlp on %s", ydl.url)
# should_download_vid = not ydl.is_youtube_host # should_download_vid = not ydl.is_youtube_host
@ -364,9 +366,9 @@ def _try_youtube_dl(worker, ydl, site, page):
# OSError('Tunnel connection failed: 464 Host Not Allowed') (caused by ProxyError...) # OSError('Tunnel connection failed: 464 Host Not Allowed') (caused by ProxyError...)
# and others... # and others...
attempt += 1 attempt += 1
if attempt == MAX_YTDLP_ATTEMPTS: if attempt == max_attempts:
logging.warning( logging.warning(
"Failed after %s attempts. Error: %s", MAX_YTDLP_ATTEMPTS, e "Failed after %s attempt(s). Error: %s", max_attempts, e
) )
raise brozzler.VideoExtractorError( raise brozzler.VideoExtractorError(
"yt-dlp hit error extracting info for %s" % ydl.url "yt-dlp hit error extracting info for %s" % ydl.url