Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 00:29:53 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

mostly black'd

Browse Source
This commit is contained in:
Barbara Miller 2024-09-18 15:58:19 -07:00
parent 27cb104b45
commit 2aa1788637
1 changed files with 28 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
brozzler/ydl.py
View File

@ -34,7 +34,7 @@ import time
thread_local = threading.local() thread_local = threading.local()
PROXYRACK_PROXY = "@@@" PROXYRACK_PROXY = "@@@"
MAX_YTDLP_ATTEMPTS = 3 MAX_YTDLP_ATTEMPTS = 4
YTDLP_WAIT = 10 YTDLP_WAIT = 10
@ -184,6 +184,7 @@ def _build_youtube_dl(worker, destdir, site, page):
worker._proxy_for(site), worker._proxy_for(site),
url, url,
) )
try:
with open(info_dict["filepath"], "rb") as f: with open(info_dict["filepath"], "rb") as f:
# include content-length header to avoid chunked # include content-length header to avoid chunked
# transfer, which warcprox currently rejects # transfer, which warcprox currently rejects
@ -206,6 +207,8 @@ def _build_youtube_dl(worker, destdir, site, page):
"content-length": size, "content-length": size,
} }
) )
except:
traceback.print_exc()
def maybe_heartbeat_site_last_claimed(*args, **kwargs): def maybe_heartbeat_site_last_claimed(*args, **kwargs):
# in case yt-dlp takes a long time, heartbeat site.last_claimed # in case yt-dlp takes a long time, heartbeat site.last_claimed
@ -306,7 +309,9 @@ def _remember_videos(page, pushed_videos=None):
def _try_youtube_dl(worker, ydl, site, page): def _try_youtube_dl(worker, ydl, site, page):
ytdlp_url = page.redirect_url if page.redirect_url else page.url ytdlp_url = page.redirect_url if page.redirect_url else page.url
youtube_host = "youtube.com" in ytdlp_url.split("//")[-1].split("/")[0].split("?")[0] youtube_host = (
"youtube.com" in ytdlp_url.split("//")[-1].split("/")[0].split("?")[0]
)
attempt = 0 attempt = 0
while attempt < MAX_YTDLP_ATTEMPTS: while attempt < MAX_YTDLP_ATTEMPTS:
try: try: