Matrix-Mirrors/anonymousland-synapse
1
0
Fork 0
mirror of https://git.anonymousland.org/anonymousland/synapse.git synced 2024-12-28 11:19:22 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1101 from matrix-org/erikj/state_types_idx

Browse Source 
Change state fetch query for postgres to be faster
This commit is contained in:
Erik Johnston 2016-09-12 10:20:38 +01:00 committed by GitHub
commit 45dc260060
1 changed files with 34 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
synapse/storage/state.py
View File

@ -306,13 +306,6 @@ class StateStore(SQLBaseStore):
defer.returnValue(results) defer.returnValue(results)
def _get_state_groups_from_groups_txn(self, txn, groups, types=None): def _get_state_groups_from_groups_txn(self, txn, groups, types=None):
if types is not None:
where_clause = "AND (%s)" % (
" OR ".join(["(type = ? AND state_key = ?)"] * len(types)),
)
else:
where_clause = ""
results = {group: {} for group in groups} results = {group: {} for group in groups}
if isinstance(self.database_engine, PostgresEngine): if isinstance(self.database_engine, PostgresEngine):
# Temporarily disable sequential scans in this transaction. This is # Temporarily disable sequential scans in this transaction. This is
@ -342,20 +335,43 @@ class StateStore(SQLBaseStore):
WHERE state_group IN ( WHERE state_group IN (
SELECT state_group FROM state SELECT state_group FROM state
) )
%s; %s
""") % (where_clause,) """)
# Turns out that postgres doesn't like doing a list of OR's and
# is about 1000x slower, so we just issue a query for each specific
# type seperately.
if types:
clause_to_args = [
(
"AND type = ? AND state_key = ?",
(etype, state_key)
)
for etype, state_key in types
]
else:
# If types is None we fetch all the state, and so just use an
# empty where clause with no extra args.
clause_to_args = [("", [])]
for where_clause, where_args in clause_to_args:
for group in groups: for group in groups:
args = [group] args = [group]
if types is not None: args.extend(where_args)
args.extend([i for typ in types for i in typ])
txn.execute(sql, args) txn.execute(sql % (where_clause,), args)
rows = self.cursor_to_dict(txn) rows = self.cursor_to_dict(txn)
for row in rows: for row in rows:
key = (row["type"], row["state_key"]) key = (row["type"], row["state_key"])
results[group][key] = row["event_id"] results[group][key] = row["event_id"]
else: else:
if types is not None:
where_clause = "AND (%s)" % (
" OR ".join(["(type = ? AND state_key = ?)"] * len(types)),
)
else:
where_clause = ""
# We don't use WITH RECURSIVE on sqlite3 as there are distributions # We don't use WITH RECURSIVE on sqlite3 as there are distributions
# that ship with an sqlite3 version that doesn't support it (e.g. wheezy) # that ship with an sqlite3 version that doesn't support it (e.g. wheezy)
for group in groups: for group in groups: