Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

set url per postprocessor

Browse Source
This commit is contained in:
Barbara Miller 2023-09-06 17:30:48 -07:00
parent c5c918bc87
commit 7a3c6d6abe
1 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
brozzler/ydl.py
View File

@ -153,8 +153,8 @@ def _build_youtube_dl(worker, destdir, site, page):
self.logger.info( self.logger.info(
'extractor %r found a download in %s', ie.IE_NAME, url) 'extractor %r found a download in %s', ie.IE_NAME, url)
def _push_stitched_up_vid_to_warcprox(self, site, info_dict): def _push_stitched_up_vid_to_warcprox(self, site, info_dict, postprocessor):
# 220211 update: does yt-dlp supply content-type? # 220211 update: does yt-dlp supply content-type? no, not as such
# XXX Don't know how to get the right content-type. Youtube-dl # XXX Don't know how to get the right content-type. Youtube-dl
# doesn't supply it. Sometimes (with --hls-prefer-native) # doesn't supply it. Sometimes (with --hls-prefer-native)
# youtube-dl produces a stitched-up video that /usr/bin/file fails # youtube-dl produces a stitched-up video that /usr/bin/file fails
@ -171,9 +171,14 @@ def _build_youtube_dl(worker, destdir, site, page):
self.logger.warning( self.logger.warning(
'guessing mimetype %s because %r', mimetype, e) 'guessing mimetype %s because %r', mimetype, e)
# watch page postprocessor is MoveFiles
if postprocessor == 'FixupM3u8':
url = 'youtube-dl:%05d:%s' % ( url = 'youtube-dl:%05d:%s' % (
info_dict.get('playlist_index') or 1, info_dict.get('playlist_index') or 1,
info_dict['webpage_url']) info_dict['webpage_url'])
else:
url = info_dict.get('url')
size = os.path.getsize(info_dict['filepath']) size = os.path.getsize(info_dict['filepath'])
self.logger.info( self.logger.info(
'pushing %r video stitched-up as %s (%s bytes) to ' 'pushing %r video stitched-up as %s (%s bytes) to '
@ -215,9 +220,10 @@ def _build_youtube_dl(worker, destdir, site, page):
if d['status'] == 'finished': if d['status'] == 'finished':
worker.logger.info('[ydl_postprocess_hook] Finished postprocessing') worker.logger.info('[ydl_postprocess_hook] Finished postprocessing')
worker.logger.info('[ydl_postprocess_hook] postprocessor: {}'.format(d['postprocessor'])) worker.logger.info('[ydl_postprocess_hook] postprocessor: {}'.format(d['postprocessor']))
#worker.logger.info('[ydl_postprocess_hook] passed params: {}'.format(d))
# if d['postprocessor'] == 'FixupM3u8' and worker._using_warcprox(site): # if d['postprocessor'] == 'FixupM3u8' and worker._using_warcprox(site):
if worker._using_warcprox(site): if worker._using_warcprox(site):
_YoutubeDL._push_stitched_up_vid_to_warcprox(_YoutubeDL, site, d['info_dict']) _YoutubeDL._push_stitched_up_vid_to_warcprox(_YoutubeDL, site, d['info_dict'], d['postprocessor'])
# default socket_timeout is 20 -- we hit it often when cluster is busy # default socket_timeout is 20 -- we hit it often when cluster is busy
ydl_opts = { ydl_opts = {