Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-09-24 22:48:33 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

more metrics

Browse source
This commit is contained in:
Barbara Miller 2024-09-12 14:27:22 -07:00
parent c82858a9f6
commit 66827fbbcf
3 changed files with 11 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

10
brozzler/metrics.py
View file

@ -15,9 +15,9 @@ except ImportError:
from prometheus_client import Counter, Gauge, Histogram, start_http_server from prometheus_client import Counter, Gauge, Histogram, start_http_server
# fmt: off # fmt: off
brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler", labelnames=["host"]) brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler")
brozzler_page_processing_duration_seconds = Histogram("brozzler_page_processing_duration_seconds", "time spent processing a page in brozzler", labelnames=["host"]) brozzler_page_processing_duration_seconds = Histogram("brozzler_page_processing_duration_seconds", "time spent processing a page in brozzler")
brozzler_outlinks_found = Counter("brozzler_outlinks_found", "number of outlinks found by brozzler", labelnames=["host"]) brozzler_outlinks_found = Counter("brozzler_outlinks_found", "number of outlinks found by brozzler")
brozzler_last_page_crawled_time = Gauge("brozzler_last_page_crawled_time", "time of last page visit") brozzler_last_page_crawled_time = Gauge("brozzler_last_page_crawled_time", "time of last page visit")
brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler") brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler")
brozzler_resources_requested = Counter("brozzler_resources_requested", "number of resources requested", labelnames=["resource_type"]) brozzler_resources_requested = Counter("brozzler_resources_requested", "number of resources requested", labelnames=["resource_type"])
@ -25,8 +25,8 @@ brozzler_resources_fetched = Counter("brozzler_resources_fetched", "number of re
brozzler_resources_size_total = Counter("brozzler_resources_size_total", "total size of resources fetched", labelnames=["resource_type"]) brozzler_resources_size_total = Counter("brozzler_resources_size_total", "total size of resources fetched", labelnames=["resource_type"])
brozzler_resources_fetch_time = Counter("brozzler_resources_fetch_time", "time spent fetching resources", labelnames=["resource_type"]) brozzler_resources_fetch_time = Counter("brozzler_resources_fetch_time", "time spent fetching resources", labelnames=["resource_type"])
brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp") brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp")
brozzler_ydl_download_attempts= Counter("brozzler_ydl_download_attempts", "count of download attempted by brozzler yt-dlp", labelnames=["host"]) brozzler_ydl_download_attempts = Counter("brozzler_ydl_download_attempts", "count of download attempted by brozzler yt-dlp", labelnames=["host"])
brozzler_ydl_download_successes= Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["host"]) brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["host"])
# fmt: on # fmt: on
if http_sd_registry: if http_sd_registry:

2
brozzler/worker.py
View file

@ -267,6 +267,7 @@ class BrozzlerWorker:
): ):
try: try:
ydl_outlinks = ydl.do_youtube_dl(self, site, page) ydl_outlinks = ydl.do_youtube_dl(self, site, page)
metrics.brozzler_ydl_urls_checked.inc(1)
outlinks.update(ydl_outlinks) outlinks.update(ydl_outlinks)
except brozzler.ReachedLimit as e: except brozzler.ReachedLimit as e:
raise raise
@ -312,6 +313,7 @@ class BrozzlerWorker:
return False return False
return True return True
@metrics.brozzler_page_processing_duration_seconds.time()
def _browse_page(self, browser, site, page, on_screenshot=None, on_request=None): def _browse_page(self, browser, site, page, on_screenshot=None, on_request=None):
def update_page_metrics(page, outlinks): def update_page_metrics(page, outlinks):
"""Update page-level Prometheus metrics.""" """Update page-level Prometheus metrics."""

5
brozzler/ydl.py
View file

@ -27,6 +27,7 @@ import os
import json import json
import doublethink import doublethink
import datetime import datetime
from . import metrics
import threading import threading
thread_local = threading.local() thread_local = threading.local()
@ -296,9 +297,10 @@ def _remember_videos(page, pushed_videos=None):
def _try_youtube_dl(worker, ydl, site, page): def _try_youtube_dl(worker, ydl, site, page):
ytdlp_url = page.redirect_url if page.redirect_url else page.url ytdlp_url = page.redirect_url if page.redirect_url else page.url
ytdlp_host = ytdlp_url.split("//")[-1].split("/")[0].split('?')[0]
try: try:
logging.info("trying yt-dlp on %s", ytdlp_url) logging.info("trying yt-dlp on %s", ytdlp_url)
metrics.brozzler_ydl_download_attempts.labels(ytdlp_host).inc(1)
with brozzler.thread_accept_exceptions(): with brozzler.thread_accept_exceptions():
# we do whatwg canonicalization here to avoid "<urlopen error # we do whatwg canonicalization here to avoid "<urlopen error
# no host given>" resulting in ProxyError # no host given>" resulting in ProxyError
@ -307,6 +309,7 @@ def _try_youtube_dl(worker, ydl, site, page):
ie_result = ydl.sanitize_info( ie_result = ydl.sanitize_info(
ydl.extract_info(str(urlcanon.whatwg(ytdlp_url))) ydl.extract_info(str(urlcanon.whatwg(ytdlp_url)))
) )
metrics.brozzler_ydl_download_successes.labels(ytdlp_host).inc(1)
_remember_videos(page, ydl.pushed_videos) _remember_videos(page, ydl.pushed_videos)
if worker._using_warcprox(site): if worker._using_warcprox(site):
info_json = json.dumps(ie_result, sort_keys=True, indent=4) info_json = json.dumps(ie_result, sort_keys=True, indent=4)