Reduce auth chains fetched during v2 state res. (#6952)

The state res v2 algorithm only cares about the difference between auth
chains, so we can pass in the known common state to the `get_auth_chain`
storage function so that it can ignore those events.
This commit is contained in:
Erik Johnston 2020-02-19 15:04:47 +00:00 committed by GitHub
parent 0001e8397e
commit 2b37eabca1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 39 additions and 13 deletions

1
changelog.d/6952.misc Normal file
View File

@ -0,0 +1 @@
Improve perf of v2 state res for large rooms.

View File

@ -16,7 +16,7 @@
import logging import logging
from collections import namedtuple from collections import namedtuple
from typing import Dict, Iterable, List, Optional from typing import Dict, Iterable, List, Optional, Set
from six import iteritems, itervalues from six import iteritems, itervalues
@ -662,7 +662,7 @@ class StateResolutionStore(object):
allow_rejected=allow_rejected, allow_rejected=allow_rejected,
) )
def get_auth_chain(self, event_ids): def get_auth_chain(self, event_ids: List[str], ignore_events: Set[str]):
"""Gets the full auth chain for a set of events (including rejected """Gets the full auth chain for a set of events (including rejected
events). events).
@ -674,11 +674,16 @@ class StateResolutionStore(object):
presence of rejected events presence of rejected events
Args: Args:
event_ids (list): The event IDs of the events to fetch the auth event_ids: The event IDs of the events to fetch the auth chain for.
chain for. Must be state events. Must be state events.
ignore_events: Set of events to exclude from the returned auth
chain.
Returns: Returns:
Deferred[list[str]]: List of event IDs of the auth chain. Deferred[list[str]]: List of event IDs of the auth chain.
""" """
return self.store.get_auth_chain_ids(event_ids, include_given=True) return self.store.get_auth_chain_ids(
event_ids, include_given=True, ignore_events=ignore_events,
)

View File

@ -248,7 +248,7 @@ def _get_auth_chain_difference(state_sets, event_map, state_res_store):
and eid not in common and eid not in common
) )
auth_chain = yield state_res_store.get_auth_chain(auth_ids) auth_chain = yield state_res_store.get_auth_chain(auth_ids, common)
auth_ids.update(auth_chain) auth_ids.update(auth_chain)
auth_sets.append(auth_ids) auth_sets.append(auth_ids)

View File

@ -14,6 +14,7 @@
# limitations under the License. # limitations under the License.
import itertools import itertools
import logging import logging
from typing import List, Optional, Set
from six.moves import range from six.moves import range
from six.moves.queue import Empty, PriorityQueue from six.moves.queue import Empty, PriorityQueue
@ -46,21 +47,37 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, SQLBas
event_ids, include_given=include_given event_ids, include_given=include_given
).addCallback(self.get_events_as_list) ).addCallback(self.get_events_as_list)
def get_auth_chain_ids(self, event_ids, include_given=False): def get_auth_chain_ids(
self,
event_ids: List[str],
include_given: bool = False,
ignore_events: Optional[Set[str]] = None,
):
"""Get auth events for given event_ids. The events *must* be state events. """Get auth events for given event_ids. The events *must* be state events.
Args: Args:
event_ids (list): state events event_ids: state events
include_given (bool): include the given events in result include_given: include the given events in result
ignore_events: Set of events to exclude from the returned auth
chain. This is useful if the caller will just discard the
given events anyway, and saves us from figuring out their auth
chains if not required.
Returns: Returns:
list of event_ids list of event_ids
""" """
return self.db.runInteraction( return self.db.runInteraction(
"get_auth_chain_ids", self._get_auth_chain_ids_txn, event_ids, include_given "get_auth_chain_ids",
self._get_auth_chain_ids_txn,
event_ids,
include_given,
ignore_events,
) )
def _get_auth_chain_ids_txn(self, txn, event_ids, include_given): def _get_auth_chain_ids_txn(self, txn, event_ids, include_given, ignore_events):
if ignore_events is None:
ignore_events = set()
if include_given: if include_given:
results = set(event_ids) results = set(event_ids)
else: else:
@ -80,6 +97,7 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, SQLBas
txn.execute(base_sql + clause, list(args)) txn.execute(base_sql + clause, list(args))
new_front.update([r[0] for r in txn]) new_front.update([r[0] for r in txn])
new_front -= ignore_events
new_front -= results new_front -= results
front = new_front front = new_front

View File

@ -603,7 +603,7 @@ class TestStateResolutionStore(object):
return {eid: self.event_map[eid] for eid in event_ids if eid in self.event_map} return {eid: self.event_map[eid] for eid in event_ids if eid in self.event_map}
def get_auth_chain(self, event_ids): def get_auth_chain(self, event_ids, ignore_events):
"""Gets the full auth chain for a set of events (including rejected """Gets the full auth chain for a set of events (including rejected
events). events).
@ -617,6 +617,8 @@ class TestStateResolutionStore(object):
Args: Args:
event_ids (list): The event IDs of the events to fetch the auth event_ids (list): The event IDs of the events to fetch the auth
chain for. Must be state events. chain for. Must be state events.
ignore_events: Set of events to exclude from the returned auth
chain.
Returns: Returns:
Deferred[list[str]]: List of event IDs of the auth chain. Deferred[list[str]]: List of event IDs of the auth chain.
@ -627,7 +629,7 @@ class TestStateResolutionStore(object):
stack = list(event_ids) stack = list(event_ids)
while stack: while stack:
event_id = stack.pop() event_id = stack.pop()
if event_id in result: if event_id in result or event_id in ignore_events:
continue continue
result.add(event_id) result.add(event_id)