From f72fa2b102c4b5d9868186854bb6b375e5479956 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 7 Oct 2024 15:01:53 -0700 Subject: [PATCH 1/4] updated prometheus metrics --- brozzler/metrics.py | 11 ++++++++--- brozzler/worker.py | 8 ++++++-- brozzler/ydl.py | 3 ++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/brozzler/metrics.py b/brozzler/metrics.py index 81239bc..8dd10c0 100644 --- a/brozzler/metrics.py +++ b/brozzler/metrics.py @@ -17,13 +17,18 @@ except ImportError: from prometheus_client import Counter, Gauge, Histogram, start_http_server # fmt: off -brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler") +brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler") brozzler_page_processing_duration_seconds = Histogram("brozzler_page_processing_duration_seconds", "time spent processing a page in brozzler") +brozzler_in_progress_headers = Gauge("brozzler_in_progress_headers", "number of headers currently processing with brozzler") +brozzler_header_processing_duration_seconds = Histogram("brozzler_header_processing_duration_seconds", "time spent processing one page's headers in brozzler") +brozzler_in_progress_browses = Gauge("brozzler_in_progress_browse", "number of pages currently browsing with brozzler") +brozzler_browsing_duration_seconds = Histogram("brozzler_browsing_duration_seconds", "time spent browsing a page in brozzler") +brozzler_in_progress_ytdlps = Gauge("brozzler_in_progress_ytdlp", "number of ytdlp sessions currently in progress with brozzler") +brozzler_ytdlp_duration_seconds = Histogram("brozzler_ytdlp_duration_seconds", "time spent running ytdlp for a page in brozzler") +brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler") brozzler_outlinks_found = Counter("brozzler_outlinks_found", "number of outlinks found by brozzler") brozzler_last_page_crawled_time = Gauge("brozzler_last_page_crawled_time", "time of last page visit, in seconds since UNIX epoch") -brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler") brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp") -brozzler_ydl_extract_attempts = Counter("brozzler_ydl_extract_attempts", "count of extracts attempted by brozzler yt-dlp", labelnames=["youtube_host"]) brozzler_ydl_extract_successes = Counter("brozzler_ydl_extract_successes", "count of extracts completed by brozzler yt-dlp", labelnames=["youtube_host"]) brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["youtube_host"]) # fmt: on diff --git a/brozzler/worker.py b/brozzler/worker.py index 6eb5872..d289911 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -251,6 +251,8 @@ class BrozzlerWorker: img.save(out, "jpeg", quality=95) return out.getbuffer() + @metrics.brozzler_page_processing_duration_seconds.time() + @metrics.brozzler_in_progress_pages.track_inprogress() def brozzle_page( self, browser, @@ -315,6 +317,8 @@ class BrozzlerWorker: ) return outlinks + @metrics.brozzler_in_progress_headers.time() + @metrics.brozzler_header_processing_duration_seconds.track_inprogress() def _get_page_headers(self, page): # bypassing warcprox, requests' stream=True defers downloading the body of the response # see https://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow @@ -334,8 +338,8 @@ class BrozzlerWorker: return False return True - @metrics.brozzler_page_processing_duration_seconds.time() - @metrics.brozzler_in_progress_pages.track_inprogress() + @metrics.brozzler_in_progress_browses.time() + @metrics.brozzler_browsing_duration_seconds.track_inprogress() def _browse_page(self, browser, site, page, on_screenshot=None, on_request=None): def update_page_metrics(page, outlinks): """Update page-level Prometheus metrics.""" diff --git a/brozzler/ydl.py b/brozzler/ydl.py index f3f21be..a72b904 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -334,7 +334,6 @@ def _try_youtube_dl(worker, ydl, site, page): # ydl.extract_info(str(urlcanon.whatwg(ydl.url)), download=should_download_vid) # if ydl.is_youtube_host and ie_result: # download_url = ie_result.get("url") - metrics.brozzler_ydl_extract_attempts.labels(ydl.is_youtube_host).inc(1) with brozzler.thread_accept_exceptions(): # we do whatwg canonicalization here to avoid "" resulting in ProxyError @@ -406,6 +405,8 @@ def _try_youtube_dl(worker, ydl, site, page): return ie_result +@metrics.brozzler_in_progress_ytdlps.time() +@metrics.brozzler_ytdlp_duration_seconds.track_inprogress() def do_youtube_dl(worker, site, page): """ Runs yt-dlp configured for `worker` and `site` to download videos from From d7c6c579083e34de6c3cf16d8fb4d886ad8ad418 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 7 Oct 2024 15:26:18 -0700 Subject: [PATCH 2/4] fix plural typo --- brozzler/metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brozzler/metrics.py b/brozzler/metrics.py index 8dd10c0..c2c7665 100644 --- a/brozzler/metrics.py +++ b/brozzler/metrics.py @@ -21,9 +21,9 @@ brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of page brozzler_page_processing_duration_seconds = Histogram("brozzler_page_processing_duration_seconds", "time spent processing a page in brozzler") brozzler_in_progress_headers = Gauge("brozzler_in_progress_headers", "number of headers currently processing with brozzler") brozzler_header_processing_duration_seconds = Histogram("brozzler_header_processing_duration_seconds", "time spent processing one page's headers in brozzler") -brozzler_in_progress_browses = Gauge("brozzler_in_progress_browse", "number of pages currently browsing with brozzler") +brozzler_in_progress_browses = Gauge("brozzler_in_progress_browses", "number of pages currently browsing with brozzler") brozzler_browsing_duration_seconds = Histogram("brozzler_browsing_duration_seconds", "time spent browsing a page in brozzler") -brozzler_in_progress_ytdlps = Gauge("brozzler_in_progress_ytdlp", "number of ytdlp sessions currently in progress with brozzler") +brozzler_in_progress_ytdlps = Gauge("brozzler_in_progress_ytdlps", "number of ytdlp sessions currently in progress with brozzler") brozzler_ytdlp_duration_seconds = Histogram("brozzler_ytdlp_duration_seconds", "time spent running ytdlp for a page in brozzler") brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler") brozzler_outlinks_found = Counter("brozzler_outlinks_found", "number of outlinks found by brozzler") From de7a325377719886fb51f3260844938592664c16 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 7 Oct 2024 15:50:32 -0700 Subject: [PATCH 3/4] histograms track_inprogress ... --- brozzler/worker.py | 8 ++++---- brozzler/ydl.py | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/brozzler/worker.py b/brozzler/worker.py index d289911..116bb75 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -317,8 +317,8 @@ class BrozzlerWorker: ) return outlinks - @metrics.brozzler_in_progress_headers.time() - @metrics.brozzler_header_processing_duration_seconds.track_inprogress() + @metrics.brozzler_header_processing_duration_seconds.time() + @metrics.brozzler_in_progress_headers.track_inprogress() def _get_page_headers(self, page): # bypassing warcprox, requests' stream=True defers downloading the body of the response # see https://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow @@ -338,8 +338,8 @@ class BrozzlerWorker: return False return True - @metrics.brozzler_in_progress_browses.time() - @metrics.brozzler_browsing_duration_seconds.track_inprogress() + @metrics.brozzler_browsing_duration_seconds.time() + @metrics.brozzler_in_progress_browses.track_inprogress() def _browse_page(self, browser, site, page, on_screenshot=None, on_request=None): def update_page_metrics(page, outlinks): """Update page-level Prometheus metrics.""" diff --git a/brozzler/ydl.py b/brozzler/ydl.py index a72b904..682b099 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -404,9 +404,8 @@ def _try_youtube_dl(worker, ydl, site, page): ) return ie_result - -@metrics.brozzler_in_progress_ytdlps.time() -@metrics.brozzler_ytdlp_duration_seconds.track_inprogress() +@metrics.brozzler_ytdlp_duration_seconds.time() +@metrics.brozzler_in_progress_ytdlps.track_inprogress() def do_youtube_dl(worker, site, page): """ Runs yt-dlp configured for `worker` and `site` to download videos from From 93d256a7b5330a7b88e14778228331dfa140a31a Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 7 Oct 2024 16:17:49 -0700 Subject: [PATCH 4/4] black'd --- brozzler/ydl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/brozzler/ydl.py b/brozzler/ydl.py index 682b099..09593f1 100644 --- a/brozzler/ydl.py +++ b/brozzler/ydl.py @@ -404,6 +404,7 @@ def _try_youtube_dl(worker, ydl, site, page): ) return ie_result + @metrics.brozzler_ytdlp_duration_seconds.time() @metrics.brozzler_in_progress_ytdlps.track_inprogress() def do_youtube_dl(worker, site, page):