Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 23:56:34 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

updates for review of PR 287

Browse Source
This commit is contained in:
Barbara Miller 2024-09-17 17:53:50 -07:00
parent cfafc560c5
commit bb1c343724
3 changed files with 15 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
brozzler/cli.py
View File

@ -236,15 +236,16 @@ def brozzle_page(argv=None):
)
arg_parser.add_argument(
"--metrics_port",
type=int,
dest="metrics_port",
default=8888,
help="Prometheus metrics port",
help="Port for brozzler's Prometheus scrape endpoint",
)
arg_parser.add_argument(
"--registry_url",
dest="registry_url",
default=None,
help="Prometheus scrape target registry URL",
help="http-sd-registry url, for Prometheus metrics discovery",
)
arg_parser.add_argument(
"--env",
@ -540,15 +541,16 @@ def brozzler_worker(argv=None):
)
arg_parser.add_argument(
"--metrics_port",
type=int,
dest="metrics_port",
default=8888,
help="Prometheus metrics port",
help="Port for brozzler's Prometheus scrape endpoint",
)
arg_parser.add_argument(
"--registry_url",
dest="registry_url",
default=None,
help="Prometheus scrape target registry URL",
help="http-sd-registry url, for Prometheus metrics discovery",
)
arg_parser.add_argument(
"--env",

10
brozzler/metrics.py
View File

@ -10,6 +10,7 @@ try:
)
from http_sd_registry.config import ClientConfig
except ImportError:
# for users without access to http_sd_registry
http_sd_registry = None
@ -19,15 +20,15 @@ from prometheus_client import Counter, Gauge, Histogram, start_http_server
brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler")
brozzler_page_processing_duration_seconds = Histogram("brozzler_page_processing_duration_seconds", "time spent processing a page in brozzler")
brozzler_outlinks_found = Counter("brozzler_outlinks_found", "number of outlinks found by brozzler")
brozzler_last_page_crawled_time = Gauge("brozzler_last_page_crawled_time", "time of last page visit")
brozzler_last_page_crawled_time = Gauge("brozzler_last_page_crawled_time", "time of last page visit, in seconds since UNIX epoch")
brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler")
brozzler_resources_requested = Counter("brozzler_resources_requested", "number of resources requested", labelnames=["resource_type"])
brozzler_resources_fetched = Counter("brozzler_resources_fetched", "number of resources fetched", labelnames=["resource_type", "status_code"])
brozzler_resources_size_total = Counter("brozzler_resources_size_total", "total size of resources fetched", labelnames=["resource_type"])
brozzler_resources_fetch_time = Counter("brozzler_resources_fetch_time", "time spent fetching resources", labelnames=["resource_type"])
brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp")
brozzler_ydl_download_attempts = Counter("brozzler_ydl_download_attempts", "count of download attempted by brozzler yt-dlp", labelnames=["host"])
brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["host"])
brozzler_ydl_download_attempts = Counter("brozzler_ydl_download_attempts", "count of download attempted by brozzler yt-dlp", labelnames=["youtube_host"])
brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["youtube_host"])
# fmt: on
@ -42,11 +43,12 @@ def register_prom_metrics(
if registry_url is None:
return
env_for_prom = None
if env == "qa":
env_for_prom = Env.qa
elif env == "prod":
env_for_prom = Env.prod
else:
env_for_prom = Env.qa
config = ClientConfig(server_url_base=registry_url)
client = Client(config)

6
brozzler/ydl.py
View File

@ -303,12 +303,12 @@ def _remember_videos(page, pushed_videos=None):
def _try_youtube_dl(worker, ydl, site, page):
ytdlp_url = page.redirect_url if page.redirect_url else page.url
ytdlp_host = ytdlp_url.split("//")[-1].split("/")[0].split("?")[0]
youtube_host = "youtube.com" in ytdlp_url.split("//")[-1].split("/")[0].split("?")[0]
attempt = 0
while attempt < MAX_YTDLP_ATTEMPTS:
try:
logging.info("trying yt-dlp on %s", ytdlp_url)
metrics.brozzler_ydl_download_attempts.labels(ytdlp_host).inc(1)
metrics.brozzler_ydl_download_attempts.labels(youtube_host).inc(1)
with brozzler.thread_accept_exceptions():
# we do whatwg canonicalization here to avoid "<urlopen error
# no host given>" resulting in ProxyError
@ -317,7 +317,7 @@ def _try_youtube_dl(worker, ydl, site, page):
ie_result = ydl.sanitize_info(
ydl.extract_info(str(urlcanon.whatwg(ytdlp_url)))
)
metrics.brozzler_ydl_download_successes.labels(ytdlp_host).inc(1)
metrics.brozzler_ydl_download_successes.labels(youtube_host).inc(1)
break
except brozzler.ShutdownRequested as e:
raise