Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-09 06:52:46 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

ruff and isort

Browse source
This commit is contained in:
Barbara Miller 2025-07-25 10:10:42 -07:00
parent 825cf70412
commit ffaec9ddda
1 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
brozzler/ydl.py
View file

@ -24,6 +24,7 @@ import tempfile
import threading import threading
import time import time
import urllib.request import urllib.request
from dataclasses import dataclass
from typing import Any, List, Optional from typing import Any, List, Optional
import doublethink import doublethink
@ -31,7 +32,6 @@ import psycopg
import structlog import structlog
import urlcanon import urlcanon
import yt_dlp import yt_dlp
from dataclasses import dataclass
from psycopg_pool import ConnectionPool, PoolTimeout from psycopg_pool import ConnectionPool, PoolTimeout
from yt_dlp.utils import ExtractorError, match_filter_func from yt_dlp.utils import ExtractorError, match_filter_func
@ -164,6 +164,7 @@ class VideoDataClient:
results = [] results = []
return results return results
""" """
def create_video_capture_record(self, video_capture_record): def create_video_capture_record(self, video_capture_record):
# NOTE: we want to do this in brozzler postcrawl for now # NOTE: we want to do this in brozzler postcrawl for now
@ -181,6 +182,7 @@ class VideoDataClient:
return results return results
""" """
def isyoutubehost(url): def isyoutubehost(url):
# split 1 splits scheme from url, split 2 splits path from hostname # split 1 splits scheme from url, split 2 splits path from hostname
return "youtube.com" in url.split("//")[-1].split("/")[0] return "youtube.com" in url.split("//")[-1].split("/")[0]
@ -429,7 +431,7 @@ def _build_youtube_dl(worker, destdir, site, page, ytdlp_proxy_endpoints):
ytdlp_url = page.redirect_url if page.redirect_url else page.url ytdlp_url = page.redirect_url if page.redirect_url else page.url
is_youtube_host = isyoutubehost(ytdlp_url) is_youtube_host = isyoutubehost(ytdlp_url)
if is_youtube_host and ytdlp_proxy_endpoints: if is_youtube_host and ytdlp_proxy_endpoints:
if 'com/watch' not in ytdlp_url: if "com/watch" not in ytdlp_url:
ydl_opts["proxy"] = ytdlp_proxy_endpoints[4] ydl_opts["proxy"] = ytdlp_proxy_endpoints[4]
else: else:
ydl_opts["proxy"] = random.choice(ytdlp_proxy_endpoints[0:4]) ydl_opts["proxy"] = random.choice(ytdlp_proxy_endpoints[0:4])