Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #253 from internetarchive/yt-dlp-timeout

Browse Source 
add socket_timeout opt for yt-dlp

Mike Wilson reviewed this via slack. We've agreed that it may be helpful to offer this setting as a command line option for brozzler, when this code is updated again.
This commit is contained in:
Barbara Miller 2023-03-15 15:54:19 -07:00 committed by GitHub
commit 4e65c2f046
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
brozzler/ydl.py
View File

@ -229,6 +229,7 @@ def _build_youtube_dl(worker, destdir, site, page):
if d['postprocessor'] == 'FixupM3u8' and worker._using_warcprox(site): if d['postprocessor'] == 'FixupM3u8' and worker._using_warcprox(site):
_YoutubeDL._push_stitched_up_vid_to_warcprox(_YoutubeDL, site, d['info_dict']) _YoutubeDL._push_stitched_up_vid_to_warcprox(_YoutubeDL, site, d['info_dict'])
# default socket_timeout is 20 -- we hit it often when cluster is busy
ydl_opts = { ydl_opts = {
"outtmpl": "{}/ydl%(autonumber)s.out".format(destdir), "outtmpl": "{}/ydl%(autonumber)s.out".format(destdir),
"retries": 1, "retries": 1,
@ -237,6 +238,7 @@ def _build_youtube_dl(worker, destdir, site, page):
"noprogress": True, "noprogress": True,
"nopart": True, "nopart": True,
"no_color": True, "no_color": True,
"socket_timeout": 40,
"progress_hooks": [maybe_heartbeat_site_last_claimed], "progress_hooks": [maybe_heartbeat_site_last_claimed],
"postprocessor_hooks": [ydl_postprocess_hook], "postprocessor_hooks": [ydl_postprocess_hook],