Trace how long it takes for the send trasaction to complete, including retrys (#5986)

changelog.d/5986.feature

@@ -0,0 +1 @@
+Trace replication send times.

synapse/http/client.py

@@ -46,6 +46,7 @@ from synapse.http import (
     redact_uri,
 )
 from synapse.logging.context import make_deferred_yieldable
+from synapse.logging.opentracing import set_tag, start_active_span, tags
 from synapse.util.async_helpers import timeout_deferred
 from synapse.util.caches import CACHE_SIZE_FACTOR
 
@@ -269,6 +270,15 @@ class SimpleHttpClient(object):
         # log request but strip `access_token` (AS requests for example include this)
         logger.info("Sending request %s %s", method, redact_uri(uri))
 
+        with start_active_span(
+            "outgoing-client-request",
+            tags={
+                tags.SPAN_KIND: tags.SPAN_KIND_RPC_CLIENT,
+                tags.HTTP_METHOD: method,
+                tags.HTTP_URL: uri,
+            },
+            finish_on_close=True,
+        ):
             try:
                 body_producer = None
                 if data is not None:
@@ -292,7 +302,10 @@ class SimpleHttpClient(object):
 
                 incoming_responses_counter.labels(method, response.code).inc()
                 logger.info(
-                "Received response to %s %s: %s", method, redact_uri(uri), response.code
+                    "Received response to %s %s: %s",
+                    method,
+                    redact_uri(uri),
+                    response.code,
                 )
                 return response
             except Exception as e:
@@ -304,6 +317,8 @@ class SimpleHttpClient(object):
                     type(e).__name__,
                     e.args[0],
                 )
+                set_tag(tags.ERROR, True)
+                set_tag("error_reason", e.args[0])
                 raise
 
     @defer.inlineCallbacks

synapse/http/matrixfederationclient.py

@@ -345,7 +345,6 @@ class MatrixFederationHttpClient(object):
         else:
             query_bytes = b""
 
-        # Retreive current span
         scope = start_active_span(
             "outgoing-federation-request",
             tags={

synapse/replication/http/_base.py

@@ -28,7 +28,11 @@ from synapse.api.errors import (
     RequestSendFailed,
     SynapseError,
 )
-from synapse.logging.opentracing import inject_active_span_byte_dict, trace_servlet
+from synapse.logging.opentracing import (
+    inject_active_span_byte_dict,
+    trace,
+    trace_servlet,
+)
 from synapse.util.caches.response_cache import ResponseCache
 from synapse.util.stringutils import random_string
 
@@ -129,6 +133,7 @@ class ReplicationEndpoint(object):
 
         client = hs.get_simple_http_client()
 
+        @trace(opname="outgoing_replication_request")
         @defer.inlineCallbacks
         def send_request(**kwargs):
             data = yield cls._serialize_payload(**kwargs)