Merge pull request #315 from galgeek/bmiller/proxy_select

yt-dlp proxy handling update

brozzler/cli.py

@@ -2,7 +2,7 @@
 """
 brozzler/cli.py - brozzler command line executables
 
-Copyright (C) 2014-2024 Internet Archive
+Copyright (C) 2014-2025 Internet Archive
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -546,6 +546,12 @@ def brozzler_worker(argv=None):
         default="/tmp",
         help="argparse.SUPPRESS",
     )
+    arg_parser.add_argument(
+        "--ytdlp_proxy_file",
+        dest="ytdlp_proxy_file",
+        default="/opt/local/brozzler/ytdlp_proxy_endpoints.txt",
+        help="argparse.SUPPRESS",
+    )
     arg_parser.add_argument(
         "--stealth",
         dest="stealth",
@@ -611,14 +617,32 @@ def brozzler_worker(argv=None):
             logging.info("running with empty skip_av_seeds")
         return skip_av_seeds
 
+    def get_ytdlp_proxy_endpoints():
+        YTDLP_PROXY_ENDPOINTS_FILE = args.ytdlp_proxy_file
+        try:
+            # make list from file
+            with open(YTDLP_PROXY_ENDPOINTS_FILE) as endpoints:
+                ytdlp_proxy_endpoints = [l for l in endpoints.readlines()]
+                if ytdlp_proxy_endpoints:
+                    logging.info(
+                        "running with ytdlp proxy endpoints file %s"
+                        % YTDLP_PROXY_ENDPOINTS_FILE
+                    )
+        except Exception as e:
+            ytdlp_proxy_endpoints = []
+            logging.info("running with empty proxy endpoints file")
+        return ytdlp_proxy_endpoints
+
     rr = rethinker(args)
     frontier = brozzler.RethinkDbFrontier(rr)
     service_registry = doublethink.ServiceRegistry(rr)
     skip_av_seeds_from_file = get_skip_av_seeds()
+    ytdlp_proxy_endpoints_from_file = get_ytdlp_proxy_endpoints()
     worker = brozzler.worker.BrozzlerWorker(
         frontier,
         service_registry,
         skip_av_seeds=skip_av_seeds_from_file,
+        ytdlp_proxy_endpoints=ytdlp_proxy_endpoints_from_file,
         max_browsers=int(args.max_browsers),
         chrome_exe=args.chrome_exe,
         proxy=args.proxy,

brozzler/worker.py

@@ -3,7 +3,7 @@ brozzler/worker.py - BrozzlerWorker brozzles pages from the frontier, meaning
 it runs yt-dlp on them, browses them and runs behaviors if appropriate,
 scopes and adds outlinks to the frontier
 
-Copyright (C) 2014-2024 Internet Archive
+Copyright (C) 2014-2025 Internet Archive
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -57,6 +57,7 @@ class BrozzlerWorker:
         frontier,
         service_registry=None,
         skip_av_seeds=None,
+        ytdlp_proxy_endpoints=None,
         max_browsers=1,
         chrome_exe="chromium-browser",
         warcprox_auto=False,
@@ -81,6 +82,7 @@ class BrozzlerWorker:
         self._frontier = frontier
         self._service_registry = service_registry
         self._skip_av_seeds = skip_av_seeds
+        self._ytdlp_proxy_endpoints = ytdlp_proxy_endpoints
         self._max_browsers = max_browsers
 
         self._warcprox_auto = warcprox_auto
@@ -290,7 +292,9 @@ class BrozzlerWorker:
                 site, page, status_code, self._skip_av_seeds
             ):
                 try:
-                    ydl_outlinks = ydl.do_youtube_dl(self, site, page)
+                    ydl_outlinks = ydl.do_youtube_dl(
+                        self, site, page, self._ytdlp_proxy_endpoints
+                    )
                     metrics.brozzler_ydl_urls_checked.inc(1)
                     outlinks.update(ydl_outlinks)
                 except brozzler.ReachedLimit as e:

brozzler/ydl.py

@@ -1,7 +1,7 @@
 """
 brozzler/ydl.py - youtube-dl / yt-dlp support for brozzler
 
-Copyright (C) 2024 Internet Archive
+Copyright (C) 2024-2025 Internet Archive
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -28,13 +28,13 @@ import json
 import doublethink
 import datetime
 from . import metrics
+import random
 import threading
 import time
 
 thread_local = threading.local()
 
 
-YTDLP_PROXY = ""
 PROXY_ATTEMPTS = 4
 YTDLP_WAIT = 10
 YTDLP_MAX_REDIRECTS = 5
@@ -91,7 +91,7 @@ class ExtraHeaderAdder(urllib.request.BaseHandler):
         return req
 
 
-def _build_youtube_dl(worker, destdir, site, page):
+def _build_youtube_dl(worker, destdir, site, page, ytdlp_proxy_endpoints):
     """
     Builds a yt-dlp `yt_dlp.YoutubeDL` for brozzling `site` with `worker`.
 
@@ -308,11 +308,11 @@ def _build_youtube_dl(worker, destdir, site, page):
 
     ytdlp_url = page.redirect_url if page.redirect_url else page.url
     is_youtube_host = isyoutubehost(ytdlp_url)
-    if is_youtube_host and YTDLP_PROXY:
-        ydl_opts["proxy"] = YTDLP_PROXY
+    if is_youtube_host and ytdlp_proxy_endpoints:
+        ydl_opts["proxy"] = random.choice(ytdlp_proxy_endpoints)
         # don't log proxy value secrets
         ytdlp_proxy_for_logs = (
-            YTDLP_PROXY.split("@")[1] if "@" in YTDLP_PROXY else "@@@"
+            ydl_opts["proxy"].split("@")[1] if "@" in ydl_opts["proxy"] else "@@@"
         )
         logging.info("using yt-dlp proxy ... %s", ytdlp_proxy_for_logs)
 
@@ -436,7 +436,7 @@ def _try_youtube_dl(worker, ydl, site, page):
 
 @metrics.brozzler_ytdlp_duration_seconds.time()
 @metrics.brozzler_in_progress_ytdlps.track_inprogress()
-def do_youtube_dl(worker, site, page):
+def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints):
     """
     Runs yt-dlp configured for `worker` and `site` to download videos from
     `page`.
@@ -453,7 +453,7 @@ def do_youtube_dl(worker, site, page):
         prefix="brzl-ydl-", dir=worker._ytdlp_tmpdir
     ) as tempdir:
         logging.info("tempdir for yt-dlp: %s", tempdir)
-        ydl = _build_youtube_dl(worker, tempdir, site, page)
+        ydl = _build_youtube_dl(worker, tempdir, site, page, ytdlp_proxy_endpoints)
         ie_result = _try_youtube_dl(worker, ydl, site, page)
         outlinks = set()
         if ie_result and (