mirror of
https://mau.dev/maunium/synapse.git
synced 2024-08-26 12:16:12 +00:00
Ensure rejected events get added to some metadata tables (#9016)
Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>
This commit is contained in:
parent
2fb1c2b6e6
commit
63f4990298
1
changelog.d/9016.misc
Normal file
1
changelog.d/9016.misc
Normal file
@ -0,0 +1 @@
|
|||||||
|
Ensure rejected events get added to some metadata tables.
|
@ -799,7 +799,8 @@ class PersistEventsStore:
|
|||||||
return [ec for ec in events_and_contexts if ec[0] not in to_remove]
|
return [ec for ec in events_and_contexts if ec[0] not in to_remove]
|
||||||
|
|
||||||
def _store_event_txn(self, txn, events_and_contexts):
|
def _store_event_txn(self, txn, events_and_contexts):
|
||||||
"""Insert new events into the event and event_json tables
|
"""Insert new events into the event, event_json, redaction and
|
||||||
|
state_events tables.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
txn (twisted.enterprise.adbapi.Connection): db connection
|
txn (twisted.enterprise.adbapi.Connection): db connection
|
||||||
@ -871,6 +872,29 @@ class PersistEventsStore:
|
|||||||
updatevalues={"have_censored": False},
|
updatevalues={"have_censored": False},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
state_events_and_contexts = [
|
||||||
|
ec for ec in events_and_contexts if ec[0].is_state()
|
||||||
|
]
|
||||||
|
|
||||||
|
state_values = []
|
||||||
|
for event, context in state_events_and_contexts:
|
||||||
|
vals = {
|
||||||
|
"event_id": event.event_id,
|
||||||
|
"room_id": event.room_id,
|
||||||
|
"type": event.type,
|
||||||
|
"state_key": event.state_key,
|
||||||
|
}
|
||||||
|
|
||||||
|
# TODO: How does this work with backfilling?
|
||||||
|
if hasattr(event, "replaces_state"):
|
||||||
|
vals["prev_state"] = event.replaces_state
|
||||||
|
|
||||||
|
state_values.append(vals)
|
||||||
|
|
||||||
|
self.db_pool.simple_insert_many_txn(
|
||||||
|
txn, table="state_events", values=state_values
|
||||||
|
)
|
||||||
|
|
||||||
def _store_rejected_events_txn(self, txn, events_and_contexts):
|
def _store_rejected_events_txn(self, txn, events_and_contexts):
|
||||||
"""Add rows to the 'rejections' table for received events which were
|
"""Add rows to the 'rejections' table for received events which were
|
||||||
rejected
|
rejected
|
||||||
@ -987,29 +1011,6 @@ class PersistEventsStore:
|
|||||||
txn, [event for event, _ in events_and_contexts]
|
txn, [event for event, _ in events_and_contexts]
|
||||||
)
|
)
|
||||||
|
|
||||||
state_events_and_contexts = [
|
|
||||||
ec for ec in events_and_contexts if ec[0].is_state()
|
|
||||||
]
|
|
||||||
|
|
||||||
state_values = []
|
|
||||||
for event, context in state_events_and_contexts:
|
|
||||||
vals = {
|
|
||||||
"event_id": event.event_id,
|
|
||||||
"room_id": event.room_id,
|
|
||||||
"type": event.type,
|
|
||||||
"state_key": event.state_key,
|
|
||||||
}
|
|
||||||
|
|
||||||
# TODO: How does this work with backfilling?
|
|
||||||
if hasattr(event, "replaces_state"):
|
|
||||||
vals["prev_state"] = event.replaces_state
|
|
||||||
|
|
||||||
state_values.append(vals)
|
|
||||||
|
|
||||||
self.db_pool.simple_insert_many_txn(
|
|
||||||
txn, table="state_events", values=state_values
|
|
||||||
)
|
|
||||||
|
|
||||||
# Prefill the event cache
|
# Prefill the event cache
|
||||||
self._add_to_cache(txn, events_and_contexts)
|
self._add_to_cache(txn, events_and_contexts)
|
||||||
|
|
||||||
|
@ -14,10 +14,15 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
from typing import List, Tuple
|
||||||
|
|
||||||
from synapse.api.constants import EventContentFields
|
from synapse.api.constants import EventContentFields
|
||||||
|
from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
|
||||||
|
from synapse.events import make_event_from_dict
|
||||||
from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause
|
from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause
|
||||||
from synapse.storage.database import DatabasePool
|
from synapse.storage.database import DatabasePool
|
||||||
|
from synapse.storage.types import Cursor
|
||||||
|
from synapse.types import JsonDict
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@ -99,6 +104,10 @@ class EventsBackgroundUpdatesStore(SQLBaseStore):
|
|||||||
columns=["user_id", "created_ts"],
|
columns=["user_id", "created_ts"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
self.db_pool.updates.register_background_update_handler(
|
||||||
|
"rejected_events_metadata", self._rejected_events_metadata,
|
||||||
|
)
|
||||||
|
|
||||||
async def _background_reindex_fields_sender(self, progress, batch_size):
|
async def _background_reindex_fields_sender(self, progress, batch_size):
|
||||||
target_min_stream_id = progress["target_min_stream_id_inclusive"]
|
target_min_stream_id = progress["target_min_stream_id_inclusive"]
|
||||||
max_stream_id = progress["max_stream_id_exclusive"]
|
max_stream_id = progress["max_stream_id_exclusive"]
|
||||||
@ -582,3 +591,118 @@ class EventsBackgroundUpdatesStore(SQLBaseStore):
|
|||||||
await self.db_pool.updates._end_background_update("event_store_labels")
|
await self.db_pool.updates._end_background_update("event_store_labels")
|
||||||
|
|
||||||
return num_rows
|
return num_rows
|
||||||
|
|
||||||
|
async def _rejected_events_metadata(self, progress: dict, batch_size: int) -> int:
|
||||||
|
"""Adds rejected events to the `state_events` and `event_auth` metadata
|
||||||
|
tables.
|
||||||
|
"""
|
||||||
|
|
||||||
|
last_event_id = progress.get("last_event_id", "")
|
||||||
|
|
||||||
|
def get_rejected_events(
|
||||||
|
txn: Cursor,
|
||||||
|
) -> List[Tuple[str, str, JsonDict, bool, bool]]:
|
||||||
|
# Fetch rejected event json, their room version and whether we have
|
||||||
|
# inserted them into the state_events or auth_events tables.
|
||||||
|
#
|
||||||
|
# Note we can assume that events that don't have a corresponding
|
||||||
|
# room version are V1 rooms.
|
||||||
|
sql = """
|
||||||
|
SELECT DISTINCT
|
||||||
|
event_id,
|
||||||
|
COALESCE(room_version, '1'),
|
||||||
|
json,
|
||||||
|
state_events.event_id IS NOT NULL,
|
||||||
|
event_auth.event_id IS NOT NULL
|
||||||
|
FROM rejections
|
||||||
|
INNER JOIN event_json USING (event_id)
|
||||||
|
LEFT JOIN rooms USING (room_id)
|
||||||
|
LEFT JOIN state_events USING (event_id)
|
||||||
|
LEFT JOIN event_auth USING (event_id)
|
||||||
|
WHERE event_id > ?
|
||||||
|
ORDER BY event_id
|
||||||
|
LIMIT ?
|
||||||
|
"""
|
||||||
|
|
||||||
|
txn.execute(sql, (last_event_id, batch_size,))
|
||||||
|
|
||||||
|
return [(row[0], row[1], db_to_json(row[2]), row[3], row[4]) for row in txn] # type: ignore
|
||||||
|
|
||||||
|
results = await self.db_pool.runInteraction(
|
||||||
|
desc="_rejected_events_metadata_get", func=get_rejected_events
|
||||||
|
)
|
||||||
|
|
||||||
|
if not results:
|
||||||
|
await self.db_pool.updates._end_background_update(
|
||||||
|
"rejected_events_metadata"
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
state_events = []
|
||||||
|
auth_events = []
|
||||||
|
for event_id, room_version, event_json, has_state, has_event_auth in results:
|
||||||
|
last_event_id = event_id
|
||||||
|
|
||||||
|
if has_state and has_event_auth:
|
||||||
|
continue
|
||||||
|
|
||||||
|
room_version_obj = KNOWN_ROOM_VERSIONS.get(room_version)
|
||||||
|
if not room_version_obj:
|
||||||
|
# We no longer support this room version, so we just ignore the
|
||||||
|
# events entirely.
|
||||||
|
logger.info(
|
||||||
|
"Ignoring event with unknown room version %r: %r",
|
||||||
|
room_version,
|
||||||
|
event_id,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
event = make_event_from_dict(event_json, room_version_obj)
|
||||||
|
|
||||||
|
if not event.is_state():
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not has_state:
|
||||||
|
state_events.append(
|
||||||
|
{
|
||||||
|
"event_id": event.event_id,
|
||||||
|
"room_id": event.room_id,
|
||||||
|
"type": event.type,
|
||||||
|
"state_key": event.state_key,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
if not has_event_auth:
|
||||||
|
for auth_id in event.auth_event_ids():
|
||||||
|
auth_events.append(
|
||||||
|
{
|
||||||
|
"room_id": event.room_id,
|
||||||
|
"event_id": event.event_id,
|
||||||
|
"auth_id": auth_id,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
if state_events:
|
||||||
|
await self.db_pool.simple_insert_many(
|
||||||
|
table="state_events",
|
||||||
|
values=state_events,
|
||||||
|
desc="_rejected_events_metadata_state_events",
|
||||||
|
)
|
||||||
|
|
||||||
|
if auth_events:
|
||||||
|
await self.db_pool.simple_insert_many(
|
||||||
|
table="event_auth",
|
||||||
|
values=auth_events,
|
||||||
|
desc="_rejected_events_metadata_event_auth",
|
||||||
|
)
|
||||||
|
|
||||||
|
await self.db_pool.updates._background_update_progress(
|
||||||
|
"rejected_events_metadata", {"last_event_id": last_event_id}
|
||||||
|
)
|
||||||
|
|
||||||
|
if len(results) < batch_size:
|
||||||
|
await self.db_pool.updates._end_background_update(
|
||||||
|
"rejected_events_metadata"
|
||||||
|
)
|
||||||
|
|
||||||
|
return len(results)
|
||||||
|
@ -0,0 +1,17 @@
|
|||||||
|
/* Copyright 2020 The Matrix.org Foundation C.I.C
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
|
||||||
|
(5828, 'rejected_events_metadata', '{}');
|
Loading…
Reference in New Issue
Block a user