Optimise room creation event lookups part 2 (#13224)

This commit is contained in:
Nick Mills-Barrett 2022-07-13 20:32:46 +02:00 committed by GitHub
parent 1d5c80b161
commit 982fe29655
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 78 additions and 19 deletions

1
changelog.d/13224.misc Normal file
View File

@ -0,0 +1 @@
Further reduce queries used sending events when creating new rooms. Contributed by Nick @ Beeper (@fizzadar).

View File

@ -889,7 +889,11 @@ class RoomCreationHandler:
# override any attempt to set room versions via the creation_content # override any attempt to set room versions via the creation_content
creation_content["room_version"] = room_version.identifier creation_content["room_version"] = room_version.identifier
last_stream_id = await self._send_events_for_new_room( (
last_stream_id,
last_sent_event_id,
depth,
) = await self._send_events_for_new_room(
requester, requester,
room_id, room_id,
preset_config=preset_config, preset_config=preset_config,
@ -905,7 +909,7 @@ class RoomCreationHandler:
if "name" in config: if "name" in config:
name = config["name"] name = config["name"]
( (
_, name_event,
last_stream_id, last_stream_id,
) = await self.event_creation_handler.create_and_send_nonmember_event( ) = await self.event_creation_handler.create_and_send_nonmember_event(
requester, requester,
@ -917,12 +921,16 @@ class RoomCreationHandler:
"content": {"name": name}, "content": {"name": name},
}, },
ratelimit=False, ratelimit=False,
prev_event_ids=[last_sent_event_id],
depth=depth,
) )
last_sent_event_id = name_event.event_id
depth += 1
if "topic" in config: if "topic" in config:
topic = config["topic"] topic = config["topic"]
( (
_, topic_event,
last_stream_id, last_stream_id,
) = await self.event_creation_handler.create_and_send_nonmember_event( ) = await self.event_creation_handler.create_and_send_nonmember_event(
requester, requester,
@ -934,7 +942,11 @@ class RoomCreationHandler:
"content": {"topic": topic}, "content": {"topic": topic},
}, },
ratelimit=False, ratelimit=False,
prev_event_ids=[last_sent_event_id],
depth=depth,
) )
last_sent_event_id = topic_event.event_id
depth += 1
# we avoid dropping the lock between invites, as otherwise joins can # we avoid dropping the lock between invites, as otherwise joins can
# start coming in and making the createRoom slow. # start coming in and making the createRoom slow.
@ -949,7 +961,7 @@ class RoomCreationHandler:
for invitee in invite_list: for invitee in invite_list:
( (
_, member_event_id,
last_stream_id, last_stream_id,
) = await self.room_member_handler.update_membership_locked( ) = await self.room_member_handler.update_membership_locked(
requester, requester,
@ -959,7 +971,11 @@ class RoomCreationHandler:
ratelimit=False, ratelimit=False,
content=content, content=content,
new_room=True, new_room=True,
prev_event_ids=[last_sent_event_id],
depth=depth,
) )
last_sent_event_id = member_event_id
depth += 1
for invite_3pid in invite_3pid_list: for invite_3pid in invite_3pid_list:
id_server = invite_3pid["id_server"] id_server = invite_3pid["id_server"]
@ -968,7 +984,10 @@ class RoomCreationHandler:
medium = invite_3pid["medium"] medium = invite_3pid["medium"]
# Note that do_3pid_invite can raise a ShadowBanError, but this was # Note that do_3pid_invite can raise a ShadowBanError, but this was
# handled above by emptying invite_3pid_list. # handled above by emptying invite_3pid_list.
last_stream_id = await self.hs.get_room_member_handler().do_3pid_invite( (
member_event_id,
last_stream_id,
) = await self.hs.get_room_member_handler().do_3pid_invite(
room_id, room_id,
requester.user, requester.user,
medium, medium,
@ -977,7 +996,11 @@ class RoomCreationHandler:
requester, requester,
txn_id=None, txn_id=None,
id_access_token=id_access_token, id_access_token=id_access_token,
prev_event_ids=[last_sent_event_id],
depth=depth,
) )
last_sent_event_id = member_event_id
depth += 1
result = {"room_id": room_id} result = {"room_id": room_id}
@ -1005,20 +1028,22 @@ class RoomCreationHandler:
power_level_content_override: Optional[JsonDict] = None, power_level_content_override: Optional[JsonDict] = None,
creator_join_profile: Optional[JsonDict] = None, creator_join_profile: Optional[JsonDict] = None,
ratelimit: bool = True, ratelimit: bool = True,
) -> int: ) -> Tuple[int, str, int]:
"""Sends the initial events into a new room. """Sends the initial events into a new room.
`power_level_content_override` doesn't apply when initial state has `power_level_content_override` doesn't apply when initial state has
power level state event content. power level state event content.
Returns: Returns:
The stream_id of the last event persisted. A tuple containing the stream ID, event ID and depth of the last
event sent to the room.
""" """
creator_id = creator.user.to_string() creator_id = creator.user.to_string()
event_keys = {"room_id": room_id, "sender": creator_id, "state_key": ""} event_keys = {"room_id": room_id, "sender": creator_id, "state_key": ""}
depth = 1
last_sent_event_id: Optional[str] = None last_sent_event_id: Optional[str] = None
def create(etype: str, content: JsonDict, **kwargs: Any) -> JsonDict: def create(etype: str, content: JsonDict, **kwargs: Any) -> JsonDict:
@ -1031,6 +1056,7 @@ class RoomCreationHandler:
async def send(etype: str, content: JsonDict, **kwargs: Any) -> int: async def send(etype: str, content: JsonDict, **kwargs: Any) -> int:
nonlocal last_sent_event_id nonlocal last_sent_event_id
nonlocal depth
event = create(etype, content, **kwargs) event = create(etype, content, **kwargs)
logger.debug("Sending %s in new room", etype) logger.debug("Sending %s in new room", etype)
@ -1047,9 +1073,11 @@ class RoomCreationHandler:
# Note: we don't pass state_event_ids here because this triggers # Note: we don't pass state_event_ids here because this triggers
# an additional query per event to look them up from the events table. # an additional query per event to look them up from the events table.
prev_event_ids=[last_sent_event_id] if last_sent_event_id else [], prev_event_ids=[last_sent_event_id] if last_sent_event_id else [],
depth=depth,
) )
last_sent_event_id = sent_event.event_id last_sent_event_id = sent_event.event_id
depth += 1
return last_stream_id return last_stream_id
@ -1075,6 +1103,7 @@ class RoomCreationHandler:
content=creator_join_profile, content=creator_join_profile,
new_room=True, new_room=True,
prev_event_ids=[last_sent_event_id], prev_event_ids=[last_sent_event_id],
depth=depth,
) )
last_sent_event_id = member_event_id last_sent_event_id = member_event_id
@ -1168,7 +1197,7 @@ class RoomCreationHandler:
content={"algorithm": RoomEncryptionAlgorithms.DEFAULT}, content={"algorithm": RoomEncryptionAlgorithms.DEFAULT},
) )
return last_sent_stream_id return last_sent_stream_id, last_sent_event_id, depth
def _generate_room_id(self) -> str: def _generate_room_id(self) -> str:
"""Generates a random room ID. """Generates a random room ID.

View File

@ -285,6 +285,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
allow_no_prev_events: bool = False, allow_no_prev_events: bool = False,
prev_event_ids: Optional[List[str]] = None, prev_event_ids: Optional[List[str]] = None,
state_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None,
depth: Optional[int] = None,
txn_id: Optional[str] = None, txn_id: Optional[str] = None,
ratelimit: bool = True, ratelimit: bool = True,
content: Optional[dict] = None, content: Optional[dict] = None,
@ -315,6 +316,9 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
prev_events are set so we need to set them ourself via this argument. prev_events are set so we need to set them ourself via this argument.
This should normally be left as None, which will cause the auth_event_ids This should normally be left as None, which will cause the auth_event_ids
to be calculated based on the room state at the prev_events. to be calculated based on the room state at the prev_events.
depth: Override the depth used to order the event in the DAG.
Should normally be set to None, which will cause the depth to be calculated
based on the prev_events.
txn_id: txn_id:
ratelimit: ratelimit:
@ -370,6 +374,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
allow_no_prev_events=allow_no_prev_events, allow_no_prev_events=allow_no_prev_events,
prev_event_ids=prev_event_ids, prev_event_ids=prev_event_ids,
state_event_ids=state_event_ids, state_event_ids=state_event_ids,
depth=depth,
require_consent=require_consent, require_consent=require_consent,
outlier=outlier, outlier=outlier,
historical=historical, historical=historical,
@ -466,6 +471,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
allow_no_prev_events: bool = False, allow_no_prev_events: bool = False,
prev_event_ids: Optional[List[str]] = None, prev_event_ids: Optional[List[str]] = None,
state_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None,
depth: Optional[int] = None,
) -> Tuple[str, int]: ) -> Tuple[str, int]:
"""Update a user's membership in a room. """Update a user's membership in a room.
@ -501,6 +507,9 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
prev_events are set so we need to set them ourself via this argument. prev_events are set so we need to set them ourself via this argument.
This should normally be left as None, which will cause the auth_event_ids This should normally be left as None, which will cause the auth_event_ids
to be calculated based on the room state at the prev_events. to be calculated based on the room state at the prev_events.
depth: Override the depth used to order the event in the DAG.
Should normally be set to None, which will cause the depth to be calculated
based on the prev_events.
Returns: Returns:
A tuple of the new event ID and stream ID. A tuple of the new event ID and stream ID.
@ -540,6 +549,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
allow_no_prev_events=allow_no_prev_events, allow_no_prev_events=allow_no_prev_events,
prev_event_ids=prev_event_ids, prev_event_ids=prev_event_ids,
state_event_ids=state_event_ids, state_event_ids=state_event_ids,
depth=depth,
) )
return result return result
@ -562,6 +572,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
allow_no_prev_events: bool = False, allow_no_prev_events: bool = False,
prev_event_ids: Optional[List[str]] = None, prev_event_ids: Optional[List[str]] = None,
state_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None,
depth: Optional[int] = None,
) -> Tuple[str, int]: ) -> Tuple[str, int]:
"""Helper for update_membership. """Helper for update_membership.
@ -599,6 +610,9 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
prev_events are set so we need to set them ourself via this argument. prev_events are set so we need to set them ourself via this argument.
This should normally be left as None, which will cause the auth_event_ids This should normally be left as None, which will cause the auth_event_ids
to be calculated based on the room state at the prev_events. to be calculated based on the room state at the prev_events.
depth: Override the depth used to order the event in the DAG.
Should normally be set to None, which will cause the depth to be calculated
based on the prev_events.
Returns: Returns:
A tuple of the new event ID and stream ID. A tuple of the new event ID and stream ID.
@ -732,6 +746,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
allow_no_prev_events=allow_no_prev_events, allow_no_prev_events=allow_no_prev_events,
prev_event_ids=prev_event_ids, prev_event_ids=prev_event_ids,
state_event_ids=state_event_ids, state_event_ids=state_event_ids,
depth=depth,
content=content, content=content,
require_consent=require_consent, require_consent=require_consent,
outlier=outlier, outlier=outlier,
@ -967,6 +982,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
ratelimit=ratelimit, ratelimit=ratelimit,
prev_event_ids=latest_event_ids, prev_event_ids=latest_event_ids,
state_event_ids=state_event_ids, state_event_ids=state_event_ids,
depth=depth,
content=content, content=content,
require_consent=require_consent, require_consent=require_consent,
outlier=outlier, outlier=outlier,
@ -1322,7 +1338,9 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
requester: Requester, requester: Requester,
txn_id: Optional[str], txn_id: Optional[str],
id_access_token: Optional[str] = None, id_access_token: Optional[str] = None,
) -> int: prev_event_ids: Optional[List[str]] = None,
depth: Optional[int] = None,
) -> Tuple[str, int]:
"""Invite a 3PID to a room. """Invite a 3PID to a room.
Args: Args:
@ -1335,9 +1353,13 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
txn_id: The transaction ID this is part of, or None if this is not txn_id: The transaction ID this is part of, or None if this is not
part of a transaction. part of a transaction.
id_access_token: The optional identity server access token. id_access_token: The optional identity server access token.
depth: Override the depth used to order the event in the DAG.
prev_event_ids: The event IDs to use as the prev events
Should normally be set to None, which will cause the depth to be calculated
based on the prev_events.
Returns: Returns:
The new stream ID. Tuple of event ID and stream ordering position
Raises: Raises:
ShadowBanError if the requester has been shadow-banned. ShadowBanError if the requester has been shadow-banned.
@ -1383,7 +1405,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
# We don't check the invite against the spamchecker(s) here (through # We don't check the invite against the spamchecker(s) here (through
# user_may_invite) because we'll do it further down the line anyway (in # user_may_invite) because we'll do it further down the line anyway (in
# update_membership_locked). # update_membership_locked).
_, stream_id = await self.update_membership( event_id, stream_id = await self.update_membership(
requester, UserID.from_string(invitee), room_id, "invite", txn_id=txn_id requester, UserID.from_string(invitee), room_id, "invite", txn_id=txn_id
) )
else: else:
@ -1402,7 +1424,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
additional_fields=spam_check[1], additional_fields=spam_check[1],
) )
stream_id = await self._make_and_store_3pid_invite( event, stream_id = await self._make_and_store_3pid_invite(
requester, requester,
id_server, id_server,
medium, medium,
@ -1411,9 +1433,12 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
inviter, inviter,
txn_id=txn_id, txn_id=txn_id,
id_access_token=id_access_token, id_access_token=id_access_token,
prev_event_ids=prev_event_ids,
depth=depth,
) )
event_id = event.event_id
return stream_id return event_id, stream_id
async def _make_and_store_3pid_invite( async def _make_and_store_3pid_invite(
self, self,
@ -1425,7 +1450,9 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
user: UserID, user: UserID,
txn_id: Optional[str], txn_id: Optional[str],
id_access_token: Optional[str] = None, id_access_token: Optional[str] = None,
) -> int: prev_event_ids: Optional[List[str]] = None,
depth: Optional[int] = None,
) -> Tuple[EventBase, int]:
room_state = await self._storage_controllers.state.get_current_state( room_state = await self._storage_controllers.state.get_current_state(
room_id, room_id,
StateFilter.from_types( StateFilter.from_types(
@ -1518,8 +1545,10 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
}, },
ratelimit=False, ratelimit=False,
txn_id=txn_id, txn_id=txn_id,
prev_event_ids=prev_event_ids,
depth=depth,
) )
return stream_id return event, stream_id
async def _is_host_in_room(self, current_state_ids: StateMap[str]) -> bool: async def _is_host_in_room(self, current_state_ids: StateMap[str]) -> bool:
# Have we just created the room, and is this about to be the very # Have we just created the room, and is this about to be the very

View File

@ -709,7 +709,7 @@ class RoomsCreateTestCase(RoomBase):
self.assertEqual(200, channel.code, channel.result) self.assertEqual(200, channel.code, channel.result)
self.assertTrue("room_id" in channel.json_body) self.assertTrue("room_id" in channel.json_body)
assert channel.resource_usage is not None assert channel.resource_usage is not None
self.assertEqual(37, channel.resource_usage.db_txn_count) self.assertEqual(32, channel.resource_usage.db_txn_count)
def test_post_room_initial_state(self) -> None: def test_post_room_initial_state(self) -> None:
# POST with initial_state config key, expect new room id # POST with initial_state config key, expect new room id
@ -722,7 +722,7 @@ class RoomsCreateTestCase(RoomBase):
self.assertEqual(200, channel.code, channel.result) self.assertEqual(200, channel.code, channel.result)
self.assertTrue("room_id" in channel.json_body) self.assertTrue("room_id" in channel.json_body)
assert channel.resource_usage is not None assert channel.resource_usage is not None
self.assertEqual(41, channel.resource_usage.db_txn_count) self.assertEqual(35, channel.resource_usage.db_txn_count)
def test_post_room_visibility_key(self) -> None: def test_post_room_visibility_key(self) -> None:
# POST with visibility config key, expect new room id # POST with visibility config key, expect new room id
@ -3283,7 +3283,7 @@ class ThreepidInviteTestCase(unittest.HomeserverTestCase):
# Mock a few functions to prevent the test from failing due to failing to talk to # Mock a few functions to prevent the test from failing due to failing to talk to
# a remote IS. We keep the mock for make_and_store_3pid_invite around so we # a remote IS. We keep the mock for make_and_store_3pid_invite around so we
# can check its call_count later on during the test. # can check its call_count later on during the test.
make_invite_mock = Mock(return_value=make_awaitable(0)) make_invite_mock = Mock(return_value=make_awaitable((Mock(event_id="abc"), 0)))
self.hs.get_room_member_handler()._make_and_store_3pid_invite = make_invite_mock self.hs.get_room_member_handler()._make_and_store_3pid_invite = make_invite_mock
self.hs.get_identity_handler().lookup_3pid = Mock( self.hs.get_identity_handler().lookup_3pid = Mock(
return_value=make_awaitable(None), return_value=make_awaitable(None),
@ -3344,7 +3344,7 @@ class ThreepidInviteTestCase(unittest.HomeserverTestCase):
# Mock a few functions to prevent the test from failing due to failing to talk to # Mock a few functions to prevent the test from failing due to failing to talk to
# a remote IS. We keep the mock for make_and_store_3pid_invite around so we # a remote IS. We keep the mock for make_and_store_3pid_invite around so we
# can check its call_count later on during the test. # can check its call_count later on during the test.
make_invite_mock = Mock(return_value=make_awaitable(0)) make_invite_mock = Mock(return_value=make_awaitable((Mock(event_id="abc"), 0)))
self.hs.get_room_member_handler()._make_and_store_3pid_invite = make_invite_mock self.hs.get_room_member_handler()._make_and_store_3pid_invite = make_invite_mock
self.hs.get_identity_handler().lookup_3pid = Mock( self.hs.get_identity_handler().lookup_3pid = Mock(
return_value=make_awaitable(None), return_value=make_awaitable(None),