Merge branch 'prom-ii' into qa

brozzler/browser.py

@@ -200,8 +200,13 @@ class WebsockReceiverThread(threading.Thread):
         # ping_timeout is used as the timeout for the call to select.select()
         # in addition to its documented purpose, and must have a value to avoid
         # hangs in certain situations
+        #
+        # skip_ut8_validation is a recommended performance improvement:
+        # https://websocket-client.readthedocs.io/en/latest/faq.html#why-is-this-library-slow
         self.websock.run_forever(
-            sockopt=((socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),), ping_timeout=0.5, skip_utf8_validation=True,
+            sockopt=((socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),),
+            ping_timeout=0.5,
+            skip_utf8_validation=True,
         )
 
     def _on_message(self, websock, message):

brozzler/metrics.py

@@ -17,13 +17,18 @@ except ImportError:
 from prometheus_client import Counter, Gauge, Histogram, start_http_server
 
 # fmt: off
-brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler")
+brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler")
 brozzler_page_processing_duration_seconds = Histogram("brozzler_page_processing_duration_seconds", "time spent processing a page in brozzler")
+brozzler_in_progress_headers = Gauge("brozzler_in_progress_headers", "number of headers currently processing with brozzler")
+brozzler_header_processing_duration_seconds = Histogram("brozzler_header_processing_duration_seconds", "time spent processing one page's headers in brozzler")
+brozzler_in_progress_browses = Gauge("brozzler_in_progress_browse", "number of pages currently browsing with brozzler")
+brozzler_browsing_duration_seconds = Histogram("brozzler_browsing_duration_seconds", "time spent browsing a page in brozzler")
+brozzler_in_progress_ytdlps = Gauge("brozzler_in_progress_ytdlp", "number of ytdlp sessions currently in progress with brozzler")
+brozzler_ytdlp_duration_seconds = Histogram("brozzler_ytdlp_duration_seconds", "time spent running ytdlp for a page in brozzler")
+brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler")
 brozzler_outlinks_found = Counter("brozzler_outlinks_found", "number of outlinks found by brozzler")
 brozzler_last_page_crawled_time = Gauge("brozzler_last_page_crawled_time", "time of last page visit, in seconds since UNIX epoch")
-brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler")
 brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp")
-brozzler_ydl_extract_attempts = Counter("brozzler_ydl_extract_attempts", "count of extracts attempted by brozzler yt-dlp", labelnames=["youtube_host"])
 brozzler_ydl_extract_successes = Counter("brozzler_ydl_extract_successes", "count of extracts completed by brozzler yt-dlp", labelnames=["youtube_host"])
 brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["youtube_host"])
 # fmt: on

brozzler/worker.py

@@ -251,6 +251,8 @@ class BrozzlerWorker:
         img.save(out, "jpeg", quality=95)
         return out.getbuffer()
 
+    @metrics.brozzler_page_processing_duration_seconds.time()
+    @metrics.brozzler_in_progress_pages.track_inprogress()
     def brozzle_page(
         self,
         browser,
@@ -325,6 +327,8 @@ class BrozzlerWorker:
                         )
         return outlinks
 
+    @metrics.brozzler_in_progress_headers.time()
+    @metrics.brozzler_header_processing_duration_seconds.track_inprogress()
     def _get_page_headers(self, page):
         # bypassing warcprox, requests' stream=True defers downloading the body of the response
         # see https://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow
@@ -360,8 +364,8 @@ class BrozzlerWorker:
             and "application/pdf" in page_headers["content-type"]
         )
 
-    @metrics.brozzler_page_processing_duration_seconds.time()
-    @metrics.brozzler_in_progress_pages.track_inprogress()
+    @metrics.brozzler_in_progress_browses.time()
+    @metrics.brozzler_browsing_duration_seconds.track_inprogress()
     def _browse_page(self, browser, site, page, on_screenshot=None, on_request=None):
         def update_page_metrics(page, outlinks):
             """Update page-level Prometheus metrics."""

brozzler/ydl.py

@@ -342,7 +342,6 @@ def _try_youtube_dl(worker, ydl, site, page):
             # ydl.extract_info(str(urlcanon.whatwg(ydl.url)), download=should_download_vid)
             # if ydl.is_youtube_host and ie_result:
             #     download_url = ie_result.get("url")
-            metrics.brozzler_ydl_extract_attempts.labels(ydl.is_youtube_host).inc(1)
             with brozzler.thread_accept_exceptions():
                 # we do whatwg canonicalization here to avoid "<urlopen error
                 # no host given>" resulting in ProxyError
@@ -414,6 +413,8 @@ def _try_youtube_dl(worker, ydl, site, page):
     return ie_result
 
 
+@metrics.brozzler_in_progress_ytdlps.time()
+@metrics.brozzler_ytdlp_duration_seconds.track_inprogress()
 def do_youtube_dl(worker, site, page):
     """
     Runs yt-dlp configured for `worker` and `site` to download videos from

setup.py

@@ -34,7 +34,7 @@ def find_package_data(package):
 
 setuptools.setup(
     name="brozzler",
-    version="1.5.55",
+    version="1.5.56",
     description="Distributed web crawling with browsers",
     url="https://github.com/internetarchive/brozzler",
     author="Noah Levitt",