Add OpenTracing for database activity. (#10113)

This adds quite a lot of OpenTracing decoration for database activity. Specifically it adds tracing at four different levels:

 * emit a span for each "interaction" - ie, the top level database function that we tend to call "transaction", but isn't really, because it can end up as multiple transactions.
 * emit a span while we hold a database connection open
 * emit a span for each database transaction - actual actual transaction.
 * emit a span for each database query.

I'm aware this might be quite a lot of overhead, but even just running it on a local Synapse it looks really interesting, and I hope the overhead can be offset just by turning down the sampling frequency and finding other ways of tracing requests of interest (eg, the `force_tracing_for_users` setting).
This commit is contained in:
Richard van der Hoff 2021-06-03 16:31:56 +01:00 committed by GitHub
parent 1d143074c5
commit 9eea4646be
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 57 additions and 30 deletions

View File

@ -0,0 +1 @@
Report OpenTracing spans for database activity.

View File

@ -271,6 +271,12 @@ class SynapseTags:
# HTTP request tag (used to distinguish full vs incremental syncs, etc) # HTTP request tag (used to distinguish full vs incremental syncs, etc)
REQUEST_TAG = "request_tag" REQUEST_TAG = "request_tag"
# Text description of a database transaction
DB_TXN_DESC = "db.txn_desc"
# Uniqueish ID of a database transaction
DB_TXN_ID = "db.txn_id"
# Block everything by default # Block everything by default
# A regex which matches the server_names to expose traces for. # A regex which matches the server_names to expose traces for.

View File

@ -40,6 +40,7 @@ from twisted.enterprise import adbapi
from synapse.api.errors import StoreError from synapse.api.errors import StoreError
from synapse.config.database import DatabaseConnectionConfig from synapse.config.database import DatabaseConnectionConfig
from synapse.logging import opentracing
from synapse.logging.context import ( from synapse.logging.context import (
LoggingContext, LoggingContext,
current_context, current_context,
@ -313,6 +314,13 @@ class LoggingTransaction:
start = time.time() start = time.time()
try: try:
with opentracing.start_active_span(
"db.query",
tags={
opentracing.tags.DATABASE_TYPE: "sql",
opentracing.tags.DATABASE_STATEMENT: sql,
},
):
return func(sql, *args) return func(sql, *args)
except Exception as e: except Exception as e:
sql_logger.debug("[SQL FAIL] {%s} %s", self.name, e) sql_logger.debug("[SQL FAIL] {%s} %s", self.name, e)
@ -525,6 +533,13 @@ class DatabasePool:
exception_callbacks=exception_callbacks, exception_callbacks=exception_callbacks,
) )
try: try:
with opentracing.start_active_span(
"db.txn",
tags={
opentracing.SynapseTags.DB_TXN_DESC: desc,
opentracing.SynapseTags.DB_TXN_ID: name,
},
):
r = func(cursor, *args, **kwargs) r = func(cursor, *args, **kwargs)
conn.commit() conn.commit()
return r return r
@ -653,6 +668,7 @@ class DatabasePool:
logger.warning("Starting db txn '%s' from sentinel context", desc) logger.warning("Starting db txn '%s' from sentinel context", desc)
try: try:
with opentracing.start_active_span(f"db.{desc}"):
result = await self.runWithConnection( result = await self.runWithConnection(
self.new_transaction, self.new_transaction,
desc, desc,
@ -718,6 +734,9 @@ class DatabasePool:
with LoggingContext( with LoggingContext(
str(curr_context), parent_context=parent_context str(curr_context), parent_context=parent_context
) as context: ) as context:
with opentracing.start_active_span(
operation_name="db.connection",
):
sched_duration_sec = monotonic_time() - start_time sched_duration_sec = monotonic_time() - start_time
sql_scheduling_timer.observe(sched_duration_sec) sql_scheduling_timer.observe(sched_duration_sec)
context.add_database_scheduled(sched_duration_sec) context.add_database_scheduled(sched_duration_sec)
@ -725,6 +744,7 @@ class DatabasePool:
if self.engine.is_connection_closed(conn): if self.engine.is_connection_closed(conn):
logger.debug("Reconnecting closed database connection") logger.debug("Reconnecting closed database connection")
conn.reconnect() conn.reconnect()
opentracing.log_kv({"message": "reconnected"})
try: try:
if db_autocommit: if db_autocommit: