Matrix-Mirrors/anonymousland-synapse
1
0
Fork 0
mirror of https://git.anonymousland.org/anonymousland/synapse.git synced 2025-05-02 22:14:55 -04:00
Code Issues Projects Releases Packages Wiki Activity

Use autocommit mode for single statement DB functions. (#8542)

Browse source 
Autocommit means that we don't wrap the functions in transactions, and instead get executed directly. Introduced in #8456. This will help:

1. reduce the number of `could not serialize access due to concurrent delete` errors that we see (though there are a few functions that often cause serialization errors that we don't fix here);
2. improve the DB performance, as it no longer needs to deal with the overhead of `REPEATABLE READ` isolation levels; and
3. improve wall clock speed of these functions, as we no longer need to send `BEGIN` and `COMMIT` to the DB.

Some notes about the differences between autocommit mode and our default `REPEATABLE READ` transactions:

1. Currently `autocommit` only applies when using PostgreSQL, and is ignored when using SQLite (due to silliness with [Twisted DB classes](https://twistedmatrix.com/trac/ticket/9998)).
2. Autocommit functions may get retried on error, which means they can get applied *twice* (or more) to the DB (since they are not in a transaction the previous call would not get rolled back). This means that the functions need to be idempotent (or otherwise not care about being called multiple times). Read queries, simple deletes, and updates/upserts that replace rows (rather than generating new values from existing rows) are all idempotent.
3. Autocommit functions no longer get executed in [`REPEATABLE READ`](https://www.postgresql.org/docs/current/transaction-iso.html) isolation level, and so data can change queries, which is fine for single statement queries.
This commit is contained in:
Erik Johnston 2020-10-14 15:50:59 +01:00 • committed by GitHub
parent 618d405a32
commit 19b15d63e8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 155 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

99
synapse/storage/database.py
View file

@ -893,6 +893,12 @@ class DatabasePool:
attempts = 0
while True:
try:
# We can autocommit if we are going to use native upserts
autocommit = (
self.engine.can_native_upsert
and table not in self._unsafe_to_upsert_tables
)
return await self.runInteraction(
desc,
self.simple_upsert_txn,
@ -901,6 +907,7 @@ class DatabasePool:
values,
insertion_values,
lock=lock,
db_autocommit=autocommit,
)
except self.engine.module.IntegrityError as e:
attempts += 1
@ -1063,6 +1070,43 @@ class DatabasePool:
)
txn.execute(sql, list(allvalues.values()))
async def simple_upsert_many(
self,
table: str,
key_names: Collection[str],
key_values: Collection[Iterable[Any]],
value_names: Collection[str],
value_values: Iterable[Iterable[Any]],
desc: str,
) -> None:
"""
Upsert, many times.
Args:
table: The table to upsert into
key_names: The key column names.
key_values: A list of each row's key column values.
value_names: The value column names
value_values: A list of each row's value column values.
Ignored if value_names is empty.
"""
# We can autocommit if we are going to use native upserts
autocommit = (
self.engine.can_native_upsert and table not in self._unsafe_to_upsert_tables
)
return await self.runInteraction(
desc,
self.simple_upsert_many_txn,
table,
key_names,
key_values,
value_names,
value_values,
db_autocommit=autocommit,
)
def simple_upsert_many_txn(
self,
txn: LoggingTransaction,
@ -1214,7 +1258,13 @@ class DatabasePool:
desc: description of the transaction, for logging and metrics
"""
return await self.runInteraction(
desc, self.simple_select_one_txn, table, keyvalues, retcols, allow_none
desc,
self.simple_select_one_txn,
table,
keyvalues,
retcols,
allow_none,
db_autocommit=True,
)
@overload
@ -1265,6 +1315,7 @@ class DatabasePool:
keyvalues,
retcol,
allow_none=allow_none,
db_autocommit=True,
)
@overload
@ -1346,7 +1397,12 @@ class DatabasePool:
Results in a list
"""
return await self.runInteraction(
desc, self.simple_select_onecol_txn, table, keyvalues, retcol
desc,
self.simple_select_onecol_txn,
table,
keyvalues,
retcol,
db_autocommit=True,
)
async def simple_select_list(
@ -1371,7 +1427,12 @@ class DatabasePool:
A list of dictionaries.
"""
return await self.runInteraction(
desc, self.simple_select_list_txn, table, keyvalues, retcols
desc,
self.simple_select_list_txn,
table,
keyvalues,
retcols,
db_autocommit=True,
)
@classmethod
@ -1450,6 +1511,7 @@ class DatabasePool:
chunk,
keyvalues,
retcols,
db_autocommit=True,
)
results.extend(rows)
@ -1548,7 +1610,12 @@ class DatabasePool:
desc: description of the transaction, for logging and metrics
"""
await self.runInteraction(
desc, self.simple_update_one_txn, table, keyvalues, updatevalues
desc,
self.simple_update_one_txn,
table,
keyvalues,
updatevalues,
db_autocommit=True,
)
@classmethod
@ -1607,7 +1674,9 @@ class DatabasePool:
keyvalues: dict of column names and values to select the row with
desc: description of the transaction, for logging and metrics
"""
await self.runInteraction(desc, self.simple_delete_one_txn, table, keyvalues)
await self.runInteraction(
desc, self.simple_delete_one_txn, table, keyvalues, db_autocommit=True,
)
@staticmethod
def simple_delete_one_txn(
@ -1646,7 +1715,9 @@ class DatabasePool:
Returns:
The number of deleted rows.
"""
return await self.runInteraction(desc, self.simple_delete_txn, table, keyvalues)
return await self.runInteraction(
desc, self.simple_delete_txn, table, keyvalues, db_autocommit=True
)
@staticmethod
def simple_delete_txn(
@ -1694,7 +1765,13 @@ class DatabasePool:
Number rows deleted
"""
return await self.runInteraction(
desc, self.simple_delete_many_txn, table, column, iterable, keyvalues
desc,
self.simple_delete_many_txn,
table,
column,
iterable,
keyvalues,
db_autocommit=True,
)
@staticmethod
@ -1860,7 +1937,13 @@ class DatabasePool:
"""
return await self.runInteraction(
desc, self.simple_search_list_txn, table, term, col, retcols
desc,
self.simple_search_list_txn,
table,
term,
col,
retcols,
db_autocommit=True,
)
@classmethod