Matrix-Mirrors/anonymousland-synapse
1
0
Fork 0
mirror of https://git.anonymousland.org/anonymousland/synapse.git synced 2025-05-02 10:46:06 -04:00
Code Issues Projects Releases Packages Wiki Activity

Refactor state group lookup to reduce DB hits (#4011)

Browse source 
Currently when fetching state groups from the data store we make two
hits two the database: once for members and once for non-members (unless
request is filtered to one or the other). This adds needless load to the
datbase, so this PR refactors the lookup to make only a single database
hit.
This commit is contained in:
Erik Johnston 2018-10-25 17:49:55 +01:00 • committed by GitHub
parent e5da60d75d
commit cb53ce9d64
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 717 additions and 488 deletions
Download patch file Download diff file Expand all files Collapse all files

175
tests/storage/test_state.py
View file

@ -18,6 +18,7 @@ import logging
from twisted.internet import defer
from synapse.api.constants import EventTypes, Membership
from synapse.storage.state import StateFilter
from synapse.types import RoomID, UserID
import tests.unittest
@ -148,7 +149,7 @@ class StateStoreTestCase(tests.unittest.TestCase):
# check we get the full state as of the final event
state = yield self.store.get_state_for_event(
e5.event_id, None, filtered_types=None
e5.event_id,
)
self.assertIsNotNone(e4)
@ -166,33 +167,35 @@ class StateStoreTestCase(tests.unittest.TestCase):
# check we can filter to the m.room.name event (with a '' state key)
state = yield self.store.get_state_for_event(
e5.event_id, [(EventTypes.Name, '')], filtered_types=None
e5.event_id, StateFilter.from_types([(EventTypes.Name, '')])
)
self.assertStateMapEqual({(e2.type, e2.state_key): e2}, state)
# check we can filter to the m.room.name event (with a wildcard None state key)
state = yield self.store.get_state_for_event(
e5.event_id, [(EventTypes.Name, None)], filtered_types=None
e5.event_id, StateFilter.from_types([(EventTypes.Name, None)])
)
self.assertStateMapEqual({(e2.type, e2.state_key): e2}, state)
# check we can grab the m.room.member events (with a wildcard None state key)
state = yield self.store.get_state_for_event(
e5.event_id, [(EventTypes.Member, None)], filtered_types=None
e5.event_id, StateFilter.from_types([(EventTypes.Member, None)])
)
self.assertStateMapEqual(
{(e3.type, e3.state_key): e3, (e5.type, e5.state_key): e5}, state
)
# check we can use filtered_types to grab a specific room member
# without filtering out the other event types
# check we can grab a specific room member without filtering out the
# other event types
state = yield self.store.get_state_for_event(
e5.event_id,
[(EventTypes.Member, self.u_alice.to_string())],
filtered_types=[EventTypes.Member],
state_filter=StateFilter(
types={EventTypes.Member: {self.u_alice.to_string()}},
include_others=True,
)
)
self.assertStateMapEqual(
@ -204,10 +207,12 @@ class StateStoreTestCase(tests.unittest.TestCase):
state,
)
# check that types=[], filtered_types=[EventTypes.Member]
# doesn't return all members
# check that we can grab everything except members
state = yield self.store.get_state_for_event(
e5.event_id, [], filtered_types=[EventTypes.Member]
e5.event_id, state_filter=StateFilter(
types={EventTypes.Member: set()},
include_others=True,
),
)
self.assertStateMapEqual(
@ -215,16 +220,21 @@ class StateStoreTestCase(tests.unittest.TestCase):
)
#######################################################
# _get_some_state_from_cache tests against a full cache
# _get_state_for_group_using_cache tests against a full cache
#######################################################
room_id = self.room.to_string()
group_ids = yield self.store.get_state_groups_ids(room_id, [e5.event_id])
group = list(group_ids.keys())[0]
# test _get_some_state_from_cache correctly filters out members with types=[]
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
self.store._state_group_cache, group, [], filtered_types=[EventTypes.Member]
# test _get_state_for_group_using_cache correctly filters out members
# with types=[]
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_cache, group,
state_filter=StateFilter(
types={EventTypes.Member: set()},
include_others=True,
),
)
self.assertEqual(is_all, True)
@ -236,22 +246,27 @@ class StateStoreTestCase(tests.unittest.TestCase):
state_dict,
)
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_members_cache,
group,
[],
filtered_types=[EventTypes.Member],
state_filter=StateFilter(
types={EventTypes.Member: set()},
include_others=True,
),
)
self.assertEqual(is_all, True)
self.assertDictEqual({}, state_dict)
# test _get_some_state_from_cache correctly filters in members with wildcard types
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
# test _get_state_for_group_using_cache correctly filters in members
# with wildcard types
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_cache,
group,
[(EventTypes.Member, None)],
filtered_types=[EventTypes.Member],
state_filter=StateFilter(
types={EventTypes.Member: None},
include_others=True,
),
)
self.assertEqual(is_all, True)
@ -263,11 +278,13 @@ class StateStoreTestCase(tests.unittest.TestCase):
state_dict,
)
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_members_cache,
group,
[(EventTypes.Member, None)],
filtered_types=[EventTypes.Member],
state_filter=StateFilter(
types={EventTypes.Member: None},
include_others=True,
),
)
self.assertEqual(is_all, True)
@ -280,12 +297,15 @@ class StateStoreTestCase(tests.unittest.TestCase):
state_dict,
)
# test _get_some_state_from_cache correctly filters in members with specific types
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
# test _get_state_for_group_using_cache correctly filters in members
# with specific types
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_cache,
group,
[(EventTypes.Member, e5.state_key)],
filtered_types=[EventTypes.Member],
state_filter=StateFilter(
types={EventTypes.Member: {e5.state_key}},
include_others=True,
),
)
self.assertEqual(is_all, True)
@ -297,23 +317,27 @@ class StateStoreTestCase(tests.unittest.TestCase):
state_dict,
)
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_members_cache,
group,
[(EventTypes.Member, e5.state_key)],
filtered_types=[EventTypes.Member],
state_filter=StateFilter(
types={EventTypes.Member: {e5.state_key}},
include_others=True,
),
)
self.assertEqual(is_all, True)
self.assertDictEqual({(e5.type, e5.state_key): e5.event_id}, state_dict)
# test _get_some_state_from_cache correctly filters in members with specific types
# and no filtered_types
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
# test _get_state_for_group_using_cache correctly filters in members
# with specific types
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_members_cache,
group,
[(EventTypes.Member, e5.state_key)],
filtered_types=None,
state_filter=StateFilter(
types={EventTypes.Member: {e5.state_key}},
include_others=False,
),
)
self.assertEqual(is_all, True)
@ -357,42 +381,54 @@ class StateStoreTestCase(tests.unittest.TestCase):
############################################
# test that things work with a partial cache
# test _get_some_state_from_cache correctly filters out members with types=[]
# test _get_state_for_group_using_cache correctly filters out members
# with types=[]
room_id = self.room.to_string()
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
self.store._state_group_cache, group, [], filtered_types=[EventTypes.Member]
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_cache, group,
state_filter=StateFilter(
types={EventTypes.Member: set()},
include_others=True,
),
)
self.assertEqual(is_all, False)
self.assertDictEqual({(e1.type, e1.state_key): e1.event_id}, state_dict)
room_id = self.room.to_string()
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_members_cache,
group,
[],
filtered_types=[EventTypes.Member],
state_filter=StateFilter(
types={EventTypes.Member: set()},
include_others=True,
),
)
self.assertEqual(is_all, True)
self.assertDictEqual({}, state_dict)
# test _get_some_state_from_cache correctly filters in members wildcard types
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
# test _get_state_for_group_using_cache correctly filters in members
# wildcard types
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_cache,
group,
[(EventTypes.Member, None)],
filtered_types=[EventTypes.Member],
state_filter=StateFilter(
types={EventTypes.Member: None},
include_others=True,
),
)
self.assertEqual(is_all, False)
self.assertDictEqual({(e1.type, e1.state_key): e1.event_id}, state_dict)
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_members_cache,
group,
[(EventTypes.Member, None)],
filtered_types=[EventTypes.Member],
state_filter=StateFilter(
types={EventTypes.Member: None},
include_others=True,
),
)
self.assertEqual(is_all, True)
@ -404,44 +440,53 @@ class StateStoreTestCase(tests.unittest.TestCase):
state_dict,
)
# test _get_some_state_from_cache correctly filters in members with specific types
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
# test _get_state_for_group_using_cache correctly filters in members
# with specific types
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_cache,
group,
[(EventTypes.Member, e5.state_key)],
filtered_types=[EventTypes.Member],
state_filter=StateFilter(
types={EventTypes.Member: {e5.state_key}},
include_others=True,
),
)
self.assertEqual(is_all, False)
self.assertDictEqual({(e1.type, e1.state_key): e1.event_id}, state_dict)
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_members_cache,
group,
[(EventTypes.Member, e5.state_key)],
filtered_types=[EventTypes.Member],
state_filter=StateFilter(
types={EventTypes.Member: {e5.state_key}},
include_others=True,
),
)
self.assertEqual(is_all, True)
self.assertDictEqual({(e5.type, e5.state_key): e5.event_id}, state_dict)
# test _get_some_state_from_cache correctly filters in members with specific types
# and no filtered_types
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
# test _get_state_for_group_using_cache correctly filters in members
# with specific types
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_cache,
group,
[(EventTypes.Member, e5.state_key)],
filtered_types=None,
state_filter=StateFilter(
types={EventTypes.Member: {e5.state_key}},
include_others=False,
),
)
self.assertEqual(is_all, False)
self.assertDictEqual({}, state_dict)
(state_dict, is_all) = yield self.store._get_some_state_from_cache(
(state_dict, is_all) = yield self.store._get_state_for_group_using_cache(
self.store._state_group_members_cache,
group,
[(EventTypes.Member, e5.state_key)],
filtered_types=None,
state_filter=StateFilter(
types={EventTypes.Member: {e5.state_key}},
include_others=False,
),
)
self.assertEqual(is_all, True)