Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-12-12 23:35:39 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

import yt-dlp directly, skip overriding urlopen

Browse source
This commit is contained in:
Barbara Miller 2023-10-20 14:41:05 -07:00
parent 6c4a9c0f89
commit 69b8c6639c
1 changed files with 6 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

20
brozzler/ydl.py
View file

@ -17,7 +17,7 @@ limitations under the License.
''' '''
import logging import logging
import yt_dlp as youtube_dl import yt_dlp
from yt_dlp.utils import match_filter_func from yt_dlp.utils import match_filter_func
import brozzler import brozzler
import urllib.request import urllib.request
@ -93,7 +93,7 @@ def final_bounces(fetches, url):
def _build_youtube_dl(worker, destdir, site, page): def _build_youtube_dl(worker, destdir, site, page):
''' '''
Builds a yt-dlp `youtube_dl.YoutubeDL` for brozzling `site` with `worker`. Builds a yt-dlp `yt_dlp.YoutubeDL` for brozzling `site` with `worker`.
The `YoutubeDL` instance does a few special brozzler-specific things: The `YoutubeDL` instance does a few special brozzler-specific things:
@ -108,20 +108,12 @@ def _build_youtube_dl(worker, destdir, site, page):
site (brozzler.Site): the site we are brozzling site (brozzler.Site): the site we are brozzling
Returns: Returns:
a yt-dlp `youtube_dl.YoutubeDL` instance a yt-dlp `yt_dlp.YoutubeDL` instance
''' '''
class _YoutubeDL(youtube_dl.YoutubeDL): class _YoutubeDL(yt_dlp.YoutubeDL):
logger = logging.getLogger(__module__ + "." + __qualname__) logger = logging.getLogger(__module__ + "." + __qualname__)
def urlopen(self, req):
try:
url = req.full_url
except AttributeError:
url = req
self.logger.debug('fetching %r', url)
return super().urlopen(req)
def add_default_extra_info(self, ie_result, ie, url): def add_default_extra_info(self, ie_result, ie, url):
# hook in some logging # hook in some logging
super().add_default_extra_info(ie_result, ie, url) super().add_default_extra_info(ie_result, ie, url)
@ -260,8 +252,8 @@ def _build_youtube_dl(worker, destdir, site, page):
# this looked like a problem with nsf-mounted homedir, shouldn't be a problem for brozzler on focal? # this looked like a problem with nsf-mounted homedir, shouldn't be a problem for brozzler on focal?
"cache_dir": "/home/archiveit", "cache_dir": "/home/archiveit",
"logger": logging.getLogger("youtube_dl"), "logger": logging.getLogger("yt_dlp"),
"verbose": True, "verbose": False,
"quiet": False, "quiet": False,
} }