Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-09-23 14:14:43 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

tidy psycopg imports, mostly

Browse source
This commit is contained in:
Barbara Miller 2025-07-30 12:26:52 -07:00
parent 782cd646f5
commit cb3d494be2
2 changed files with 2 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

7
brozzler/ydl.py
View file

@ -28,7 +28,6 @@ from dataclasses import dataclass
from typing import Any, List, Optional from typing import Any, List, Optional
import doublethink import doublethink
import psycopg
import structlog import structlog
import urlcanon import urlcanon
import yt_dlp import yt_dlp
@ -439,6 +438,7 @@ def _build_youtube_dl(worker, destdir, site, page, ytdlp_proxy_endpoints):
logger.info("using yt-dlp proxy ...", proxy=ytdlp_proxy_for_logs) logger.info("using yt-dlp proxy ...", proxy=ytdlp_proxy_for_logs)
# skip warcprox proxying yt-dlp v.2023.07.06: youtube extractor using ranges # skip warcprox proxying yt-dlp v.2023.07.06: youtube extractor using ranges
# should_proxy_vid_maybe = not ydl.is_youtube_host
# if worker._proxy_for(site): # if worker._proxy_for(site):
# ydl_opts["proxy"] = "http://{}".format(worker._proxy_for(site)) # ydl_opts["proxy"] = "http://{}".format(worker._proxy_for(site))
@ -481,11 +481,6 @@ def _try_youtube_dl(worker, ydl, site, page):
while attempt < max_attempts: while attempt < max_attempts:
try: try:
logger.info("trying yt-dlp", url=ydl.url) logger.info("trying yt-dlp", url=ydl.url)
# should_download_vid = not ydl.is_youtube_host
# then
# ydl.extract_info(str(urlcanon.whatwg(ydl.url)), download=should_download_vid)
# if ydl.is_youtube_host and ie_result:
# download_url = ie_result.get("url")
with brozzler.thread_accept_exceptions(): with brozzler.thread_accept_exceptions():
# we do whatwg canonicalization here to avoid "<urlopen error # we do whatwg canonicalization here to avoid "<urlopen error
# no host given>" resulting in ProxyError # no host given>" resulting in ProxyError

2
pyproject.toml
View file

@ -40,7 +40,7 @@ license = "Apache-2.0"
[project.optional-dependencies] [project.optional-dependencies]
yt-dlp = ["yt-dlp[default,curl-cffi]>=2024.7.25"] yt-dlp = ["yt-dlp[default,curl-cffi]>=2024.7.25"]
psycopg = ["psycopg[binary,pool]>=3.2.6"] psycopg = ["psycopg[pool]>=3.2.6"]
dashboard = ["flask>=1.0", "gunicorn>=19.8.1"] dashboard = ["flask>=1.0", "gunicorn>=19.8.1"]
warcprox = ["warcprox>=2.4.31"] warcprox = ["warcprox>=2.4.31"]
rethinkdb = [ rethinkdb = [