From 1fa2e58772620199075a36c237dd83cd989c0e91 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 7 Oct 2022 13:35:44 +0100 Subject: [PATCH] Catch BrokenPipeError from metrics server, and log as a warning (#14072) --- changelog.d/14072.misc | 1 + synapse/metrics/_legacy_exposition.py | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 changelog.d/14072.misc diff --git a/changelog.d/14072.misc b/changelog.d/14072.misc new file mode 100644 index 000000000..3070c756d --- /dev/null +++ b/changelog.d/14072.misc @@ -0,0 +1 @@ +Don't create noisy Sentry events when a requester drops connection to the metrics server mid-request. diff --git a/synapse/metrics/_legacy_exposition.py b/synapse/metrics/_legacy_exposition.py index 563d8cc2c..1459f9d22 100644 --- a/synapse/metrics/_legacy_exposition.py +++ b/synapse/metrics/_legacy_exposition.py @@ -20,7 +20,7 @@ Due to the renaming of metrics in prometheus_client 0.4.0, this customised vendoring of the code will emit both the old versions that Synapse dashboards expect, and the newer "best practice" version of the up-to-date official client. """ - +import logging import math import threading from http.server import BaseHTTPRequestHandler, HTTPServer @@ -34,6 +34,7 @@ from prometheus_client.core import Sample from twisted.web.resource import Resource from twisted.web.server import Request +logger = logging.getLogger(__name__) CONTENT_TYPE_LATEST = "text/plain; version=0.0.4; charset=utf-8" @@ -219,11 +220,16 @@ class MetricsHandler(BaseHTTPRequestHandler): except Exception: self.send_error(500, "error generating metric output") raise - self.send_response(200) - self.send_header("Content-Type", CONTENT_TYPE_LATEST) - self.send_header("Content-Length", str(len(output))) - self.end_headers() - self.wfile.write(output) + try: + self.send_response(200) + self.send_header("Content-Type", CONTENT_TYPE_LATEST) + self.send_header("Content-Length", str(len(output))) + self.end_headers() + self.wfile.write(output) + except BrokenPipeError as e: + logger.warning( + "BrokenPipeError when serving metrics (%s). Did Prometheus restart?", e + ) def log_message(self, format: str, *args: Any) -> None: """Log nothing."""