Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-12-16 00:53:49 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #293 from galgeek/prom-ii

Browse source 
update prometheus metrics, approved separately by @avdempsey
This commit is contained in:
Barbara Miller 2024-10-07 16:20:22 -07:00 • committed by GitHub
commit ed906b4aa2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 16 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

11
brozzler/metrics.py
View file

@ -17,13 +17,18 @@ except ImportError:
from prometheus_client import Counter, Gauge, Histogram, start_http_server from prometheus_client import Counter, Gauge, Histogram, start_http_server
# fmt: off # fmt: off
brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler") brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler")
brozzler_page_processing_duration_seconds = Histogram("brozzler_page_processing_duration_seconds", "time spent processing a page in brozzler") brozzler_page_processing_duration_seconds = Histogram("brozzler_page_processing_duration_seconds", "time spent processing a page in brozzler")
brozzler_in_progress_headers = Gauge("brozzler_in_progress_headers", "number of headers currently processing with brozzler")
brozzler_header_processing_duration_seconds = Histogram("brozzler_header_processing_duration_seconds", "time spent processing one page's headers in brozzler")
brozzler_in_progress_browses = Gauge("brozzler_in_progress_browses", "number of pages currently browsing with brozzler")
brozzler_browsing_duration_seconds = Histogram("brozzler_browsing_duration_seconds", "time spent browsing a page in brozzler")
brozzler_in_progress_ytdlps = Gauge("brozzler_in_progress_ytdlps", "number of ytdlp sessions currently in progress with brozzler")
brozzler_ytdlp_duration_seconds = Histogram("brozzler_ytdlp_duration_seconds", "time spent running ytdlp for a page in brozzler")
brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler")
brozzler_outlinks_found = Counter("brozzler_outlinks_found", "number of outlinks found by brozzler") brozzler_outlinks_found = Counter("brozzler_outlinks_found", "number of outlinks found by brozzler")
brozzler_last_page_crawled_time = Gauge("brozzler_last_page_crawled_time", "time of last page visit, in seconds since UNIX epoch") brozzler_last_page_crawled_time = Gauge("brozzler_last_page_crawled_time", "time of last page visit, in seconds since UNIX epoch")
brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler")
brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp") brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp")
brozzler_ydl_extract_attempts = Counter("brozzler_ydl_extract_attempts", "count of extracts attempted by brozzler yt-dlp", labelnames=["youtube_host"])
brozzler_ydl_extract_successes = Counter("brozzler_ydl_extract_successes", "count of extracts completed by brozzler yt-dlp", labelnames=["youtube_host"]) brozzler_ydl_extract_successes = Counter("brozzler_ydl_extract_successes", "count of extracts completed by brozzler yt-dlp", labelnames=["youtube_host"])
brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["youtube_host"]) brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["youtube_host"])
# fmt: on # fmt: on

8
brozzler/worker.py
View file

@ -251,6 +251,8 @@ class BrozzlerWorker:
img.save(out, "jpeg", quality=95) img.save(out, "jpeg", quality=95)
return out.getbuffer() return out.getbuffer()
@metrics.brozzler_page_processing_duration_seconds.time()
@metrics.brozzler_in_progress_pages.track_inprogress()
def brozzle_page( def brozzle_page(
self, self,
browser, browser,
@ -315,6 +317,8 @@ class BrozzlerWorker:
) )
return outlinks return outlinks
@metrics.brozzler_header_processing_duration_seconds.time()
@metrics.brozzler_in_progress_headers.track_inprogress()
def _get_page_headers(self, page): def _get_page_headers(self, page):
# bypassing warcprox, requests' stream=True defers downloading the body of the response # bypassing warcprox, requests' stream=True defers downloading the body of the response
# see https://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow # see https://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow
@ -334,8 +338,8 @@ class BrozzlerWorker:
return False return False
return True return True
@metrics.brozzler_page_processing_duration_seconds.time() @metrics.brozzler_browsing_duration_seconds.time()
@metrics.brozzler_in_progress_pages.track_inprogress() @metrics.brozzler_in_progress_browses.track_inprogress()
def _browse_page(self, browser, site, page, on_screenshot=None, on_request=None): def _browse_page(self, browser, site, page, on_screenshot=None, on_request=None):
def update_page_metrics(page, outlinks): def update_page_metrics(page, outlinks):
"""Update page-level Prometheus metrics.""" """Update page-level Prometheus metrics."""

3
brozzler/ydl.py
View file

@ -334,7 +334,6 @@ def _try_youtube_dl(worker, ydl, site, page):
# ydl.extract_info(str(urlcanon.whatwg(ydl.url)), download=should_download_vid) # ydl.extract_info(str(urlcanon.whatwg(ydl.url)), download=should_download_vid)
# if ydl.is_youtube_host and ie_result: # if ydl.is_youtube_host and ie_result:
# download_url = ie_result.get("url") # download_url = ie_result.get("url")
metrics.brozzler_ydl_extract_attempts.labels(ydl.is_youtube_host).inc(1)
with brozzler.thread_accept_exceptions(): with brozzler.thread_accept_exceptions():
# we do whatwg canonicalization here to avoid "<urlopen error # we do whatwg canonicalization here to avoid "<urlopen error
# no host given>" resulting in ProxyError # no host given>" resulting in ProxyError
@ -406,6 +405,8 @@ def _try_youtube_dl(worker, ydl, site, page):
return ie_result return ie_result
@metrics.brozzler_ytdlp_duration_seconds.time()
@metrics.brozzler_in_progress_ytdlps.track_inprogress()
def do_youtube_dl(worker, site, page): def do_youtube_dl(worker, site, page):
""" """
Runs yt-dlp configured for `worker` and `site` to download videos from Runs yt-dlp configured for `worker` and `site` to download videos from