Merge branch 'bmiller/proxy_select' into qa

brozzler/cli.py

@@ -2,7 +2,7 @@
 """
 brozzler/cli.py - brozzler command line executables
 
-Copyright (C) 2014-2024 Internet Archive
+Copyright (C) 2014-2025 Internet Archive
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -598,12 +598,30 @@ def brozzler_worker(argv=None):
         finally:
             signal.signal(signal.SIGQUIT, dump_state)
 
+    def get_proxy_endpoints():
+        PROXY_ENDPOINTS_FILE = "/opt/local/brozzler/proxy_endpoints.txt"
+        try:
+            # make list from file
+            with open(PROXY_ENDPOINTS_FILE) as endpoints:
+                proxy_endpoints = [l for l in endpoints.readlines()]
+                if proxy_endpoints:
+                    logging.info(
+                        "running with proxy endpoints file %s" % PROXY_ENDPOINTS_FILE
+                    )
+        except Exception as e:
+            proxy_endpoints = []
+            logging.info("running with empty proxy endpoints file")
+        return proxy_endpoints
+
     rr = rethinker(args)
     frontier = brozzler.RethinkDbFrontier(rr)
     service_registry = doublethink.ServiceRegistry(rr)
+    skip_av_seeds_from_file = get_skip_av_seeds()
+    proxy_endpoints_from_file = get_proxy_endpoints()
     worker = brozzler.worker.BrozzlerWorker(
         frontier,
         service_registry,
+        proxy_endpoints=proxy_endpoints_from_file,
         max_browsers=int(args.max_browsers),
         chrome_exe=args.chrome_exe,
         proxy=args.proxy,

brozzler/worker.py

@@ -57,6 +57,7 @@ class BrozzlerWorker:
         self,
         frontier,
         service_registry=None,
+        proxy_endpoints=None,
         max_browsers=1,
         chrome_exe="chromium-browser",
         warcprox_auto=False,
@@ -80,6 +81,7 @@ class BrozzlerWorker:
     ):
         self._frontier = frontier
         self._service_registry = service_registry
+        self._proxy_endpoints = proxy_endpoints
         self._max_browsers = max_browsers
 
         self._warcprox_auto = warcprox_auto
@@ -297,7 +299,7 @@ class BrozzlerWorker:
                 self.logger.info("page interstitial shown (http auth): %s", page)
 
             if enable_youtube_dl and ydl.should_ytdlp(
-                site, page, status_code
+                site, page, status_code, self._proxy_endpoints
             ):
                 try:
                     ydl_outlinks = ydl.do_youtube_dl(self, site, page)

brozzler/ydl.py

@@ -38,28 +38,11 @@ import time
 thread_local = threading.local()
 
 
-YTDLP_PROXY = ""
 PROXY_ATTEMPTS = 4
 YTDLP_WAIT = 10
 YTDLP_MAX_REDIRECTS = 5
 
-
-def _timestamp4datetime(timestamp):
-    """split `timestamp` into a tuple of 6 integers.
-
-    :param timestamp: full-length timestamp
-    """
-    timestamp = timestamp[:14]
-    return (
-        int(timestamp[:-10]),
-        int(timestamp[-10:-8]),
-        int(timestamp[-8:-6]),
-        int(timestamp[-6:-4]),
-        int(timestamp[-4:-2]),
-        int(timestamp[-2:])
-        )
-
-def should_ytdlp(site, page, page_status):
+def should_ytdlp(site, page, page_status, proxy_endpoints):
     # called only after we've passed needs_browsing() check
 
     if page_status != 200:
@@ -314,11 +297,11 @@ def _build_youtube_dl(worker, destdir, site, page):
 
     ytdlp_url = page.redirect_url if page.redirect_url else page.url
     is_youtube_host = isyoutubehost(ytdlp_url)
-    if is_youtube_host and YTDLP_PROXY:
-        ydl_opts["proxy"] = YTDLP_PROXY
+    if is_youtube_host and proxy_endpoints:
+        ydl_opts["proxy"] = random.choice(proxy_endpoints)
         # don't log proxy value secrets
         ytdlp_proxy_for_logs = (
-            YTDLP_PROXY.split("@")[1] if "@" in YTDLP_PROXY else "@@@"
+            ydl_opts["proxy"].split("@")[1] if "@" in ydl_opts["proxy"] else "@@@"
         )
         logging.info("using yt-dlp proxy ... %s", ytdlp_proxy_for_logs)