diff --git a/brozzler/browser.py b/brozzler/browser.py index c873d58..9c72589 100644 --- a/brozzler/browser.py +++ b/brozzler/browser.py @@ -200,8 +200,13 @@ class WebsockReceiverThread(threading.Thread): # ping_timeout is used as the timeout for the call to select.select() # in addition to its documented purpose, and must have a value to avoid # hangs in certain situations + # + # skip_ut8_validation is a recommended performance improvement: + # https://websocket-client.readthedocs.io/en/latest/faq.html#why-is-this-library-slow self.websock.run_forever( - sockopt=((socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),), ping_timeout=0.5, skip_utf8_validation=True, + sockopt=((socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),), + ping_timeout=0.5, + skip_utf8_validation=True, ) def _on_message(self, websock, message): diff --git a/brozzler/metrics.py b/brozzler/metrics.py index 81239bc..8dd10c0 100644 --- a/brozzler/metrics.py +++ b/brozzler/metrics.py @@ -17,13 +17,18 @@ except ImportError: from prometheus_client import Counter, Gauge, Histogram, start_http_server # fmt: off -brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler") +brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler") brozzler_page_processing_duration_seconds = Histogram("brozzler_page_processing_duration_seconds", "time spent processing a page in brozzler") +brozzler_in_progress_headers = Gauge("brozzler_in_progress_headers", "number of headers currently processing with brozzler") +brozzler_header_processing_duration_seconds = Histogram("brozzler_header_processing_duration_seconds", "time spent processing one page's headers in brozzler") +brozzler_in_progress_browses = Gauge("brozzler_in_progress_browse", "number of pages currently browsing with brozzler") +brozzler_browsing_duration_seconds = Histogram("brozzler_browsing_duration_seconds", "time spent browsing a page in brozzler") +brozzler_in_progress_ytdlps = Gauge("brozzler_in_progress_ytdlp", "number of ytdlp sessions currently in progress with brozzler") +brozzler_ytdlp_duration_seconds = Histogram("brozzler_ytdlp_duration_seconds", "time spent running ytdlp for a page in brozzler") +brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler") brozzler_outlinks_found = Counter("brozzler_outlinks_found", "number of outlinks found by brozzler") brozzler_last_page_crawled_time = Gauge("brozzler_last_page_crawled_time", "time of last page visit, in seconds since UNIX epoch") -brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler") brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp") -brozzler_ydl_extract_attempts = Counter("brozzler_ydl_extract_attempts", "count of extracts attempted by brozzler yt-dlp", labelnames=["youtube_host"]) brozzler_ydl_extract_successes = Counter("brozzler_ydl_extract_successes", "count of extracts completed by brozzler yt-dlp", labelnames=["youtube_host"]) brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["youtube_host"]) # fmt: on diff --git a/brozzler/worker.py b/brozzler/worker.py index abe84de..4e4e7d7 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -251,6 +251,8 @@ class BrozzlerWorker: img.save(out, "jpeg", quality=95) return out.getbuffer() + @metrics.brozzler_page_processing_duration_seconds.time() + @metrics.brozzler_in_progress_pages.track_inprogress() def brozzle_page( self, browser, @@ -325,6 +327,8 @@ class BrozzlerWorker: ) return outlinks + @metrics.brozzler_in_progress_headers.time() + @metrics.brozzler_header_processing_duration_seconds.track_inprogress() def _get_page_headers(self, page): # bypassing warcprox, requests' stream=True defers downloading the body of the response # see https://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow @@ -360,8 +364,8 @@ class BrozzlerWorker: and "application/pdf" in page_headers["content-type"] ) - @metrics.brozzler_page_processing_duration_seconds.time() - @metrics.brozzler_in_progress_pages.track_inprogress() + @metrics.brozzler_in_progress_browses.time() + @metrics.brozzler_browsing_duration_seconds.track_inprogress() def _browse_page(self, browser, site, page, on_screenshot=None, on_request=None): def update_page_metrics(page, outlinks): """Update page-level Prometheus metrics.""" diff --git a/brozzler/ydl.py b/brozzler/ydl.py index d789892..e7e8280 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -342,7 +342,6 @@ def _try_youtube_dl(worker, ydl, site, page): # ydl.extract_info(str(urlcanon.whatwg(ydl.url)), download=should_download_vid) # if ydl.is_youtube_host and ie_result: # download_url = ie_result.get("url") - metrics.brozzler_ydl_extract_attempts.labels(ydl.is_youtube_host).inc(1) with brozzler.thread_accept_exceptions(): # we do whatwg canonicalization here to avoid "" resulting in ProxyError @@ -414,6 +413,8 @@ def _try_youtube_dl(worker, ydl, site, page): return ie_result +@metrics.brozzler_in_progress_ytdlps.time() +@metrics.brozzler_ytdlp_duration_seconds.track_inprogress() def do_youtube_dl(worker, site, page): """ Runs yt-dlp configured for `worker` and `site` to download videos from diff --git a/setup.py b/setup.py index ecb592f..0e522ae 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ def find_package_data(package): setuptools.setup( name="brozzler", - version="1.5.55", + version="1.5.56", description="Distributed web crawling with browsers", url="https://github.com/internetarchive/brozzler", author="Noah Levitt",