Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-04 20:44:20 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'predup_type_playlist' into qa

Browse source
This commit is contained in:
Barbara Miller 2025-07-21 17:20:41 -07:00
commit 2acb1a75d1
2 changed files with 10 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
brozzler/model.py
View file

@ -201,6 +201,8 @@ class Job(doublethink.Document, ElapsedMixIn):
def populate_defaults(self):
if "status" not in self:
self.status = "ACTIVE"
if "account_id" not in self:
self.account_id = None
if "pdfs_only" not in self:
self.pdfs_only = False
if "starts_and_stops" not in self:
@ -262,6 +264,8 @@ class Site(doublethink.Document, ElapsedMixIn):
self.scope = {}
if "video_capture" not in self:
self.video_capture = VideoCaptureOptions.ENABLE_VIDEO_CAPTURE.value
if "account_id" not in self:
self.account_id = None
# backward compatibility
if "surt" in self.scope:

9
brozzler/ydl.py
View file

@ -182,8 +182,8 @@ class VideoDataClient:
def isyoutubehost(url):
# split 1 splits scheme from url, split 2 splits path from hostname, split 3 splits query string on hostname
return "youtube.com" in url.split("//")[-1].split("/")[0].split("?")[0]
# split 1 splits scheme from url, split 2 splits path from hostname
return "youtube.com" in url.split("//")[-1].split("/")[0]
class ExtraHeaderAdder(urllib.request.BaseHandler):
@ -427,7 +427,10 @@ def _build_youtube_dl(worker, destdir, site, page, ytdlp_proxy_endpoints):
ytdlp_url = page.redirect_url if page.redirect_url else page.url
is_youtube_host = isyoutubehost(ytdlp_url)
if is_youtube_host and ytdlp_proxy_endpoints:
ydl_opts["proxy"] = random.choice(ytdlp_proxy_endpoints)
if 'com/watch' not in ytdlp_url:
ydl_opts["proxy"] = ytdlp_proxy_endpoints[4]
else:
ydl_opts["proxy"] = random.choice(ytdlp_proxy_endpoints[0:4])
# don't log proxy value secrets
ytdlp_proxy_for_logs = (
ydl_opts["proxy"].split("@")[1] if "@" in ydl_opts["proxy"] else "@@@"