2016-01-07 04:26:29 +00:00
|
|
|
|
# Copyright 2014-2016 OpenMarket Ltd
|
2014-10-28 16:42:35 +00:00
|
|
|
|
#
|
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
|
#
|
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
#
|
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
|
# limitations under the License.
|
2022-09-23 19:01:29 +00:00
|
|
|
|
import datetime
|
2019-09-26 10:47:53 +00:00
|
|
|
|
import itertools
|
2018-07-09 06:09:20 +00:00
|
|
|
|
import logging
|
2020-06-16 12:51:47 +00:00
|
|
|
|
from queue import Empty, PriorityQueue
|
2022-05-18 15:02:10 +00:00
|
|
|
|
from typing import (
|
|
|
|
|
TYPE_CHECKING,
|
|
|
|
|
Collection,
|
|
|
|
|
Dict,
|
|
|
|
|
Iterable,
|
|
|
|
|
List,
|
|
|
|
|
Optional,
|
2023-02-10 23:29:00 +00:00
|
|
|
|
Sequence,
|
2022-05-18 15:02:10 +00:00
|
|
|
|
Set,
|
|
|
|
|
Tuple,
|
|
|
|
|
cast,
|
|
|
|
|
)
|
2015-05-14 12:45:48 +00:00
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
import attr
|
2021-08-02 13:37:25 +00:00
|
|
|
|
from prometheus_client import Counter, Gauge
|
2021-07-01 09:18:25 +00:00
|
|
|
|
|
2023-06-16 19:12:24 +00:00
|
|
|
|
from synapse.api.constants import MAX_DEPTH
|
2018-07-09 06:09:20 +00:00
|
|
|
|
from synapse.api.errors import StoreError
|
2021-08-02 13:37:25 +00:00
|
|
|
|
from synapse.api.room_versions import EventFormatVersions, RoomVersion
|
2021-06-29 18:55:22 +00:00
|
|
|
|
from synapse.events import EventBase, make_event_from_dict
|
2022-08-15 18:41:23 +00:00
|
|
|
|
from synapse.logging.opentracing import tag_args, trace
|
2020-10-09 11:37:51 +00:00
|
|
|
|
from synapse.metrics.background_process_metrics import wrap_as_background_process
|
2021-06-29 18:55:22 +00:00
|
|
|
|
from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause
|
2023-07-05 09:43:19 +00:00
|
|
|
|
from synapse.storage.background_updates import ForeignKeyConstraint
|
2021-12-13 17:05:00 +00:00
|
|
|
|
from synapse.storage.database import (
|
|
|
|
|
DatabasePool,
|
|
|
|
|
LoggingDatabaseConnection,
|
|
|
|
|
LoggingTransaction,
|
|
|
|
|
)
|
2020-08-05 20:38:57 +00:00
|
|
|
|
from synapse.storage.databases.main.events_worker import EventsWorkerStore
|
|
|
|
|
from synapse.storage.databases.main.signatures import SignatureWorkerStore
|
2022-09-23 19:01:29 +00:00
|
|
|
|
from synapse.storage.engines import PostgresEngine, Sqlite3Engine
|
2023-05-25 04:22:24 +00:00
|
|
|
|
from synapse.types import JsonDict, StrCollection
|
2021-06-29 18:55:22 +00:00
|
|
|
|
from synapse.util import json_encoder
|
2015-08-11 16:59:32 +00:00
|
|
|
|
from synapse.util.caches.descriptors import cached
|
2020-11-13 11:29:18 +00:00
|
|
|
|
from synapse.util.caches.lrucache import LruCache
|
2022-09-07 11:03:32 +00:00
|
|
|
|
from synapse.util.cancellation import cancellable
|
2020-02-19 15:47:11 +00:00
|
|
|
|
from synapse.util.iterutils import batch_iter
|
2014-10-28 16:42:35 +00:00
|
|
|
|
|
2021-10-22 17:15:41 +00:00
|
|
|
|
if TYPE_CHECKING:
|
|
|
|
|
from synapse.server import HomeServer
|
|
|
|
|
|
2021-07-01 09:18:25 +00:00
|
|
|
|
oldest_pdu_in_federation_staging = Gauge(
|
|
|
|
|
"synapse_federation_server_oldest_inbound_pdu_in_staging",
|
|
|
|
|
"The age in seconds since we received the oldest pdu in the federation staging area",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
number_pdus_in_federation_queue = Gauge(
|
|
|
|
|
"synapse_federation_server_number_inbound_pdu_in_staging",
|
|
|
|
|
"The total number of events in the inbound federation staging",
|
|
|
|
|
)
|
|
|
|
|
|
2021-08-02 13:37:25 +00:00
|
|
|
|
pdus_pruned_from_federation_queue = Counter(
|
|
|
|
|
"synapse_federation_server_number_inbound_pdu_pruned",
|
|
|
|
|
"The number of events in the inbound federation staging that have been "
|
|
|
|
|
"pruned due to the queue getting too long",
|
|
|
|
|
)
|
|
|
|
|
|
2014-10-28 16:42:35 +00:00
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
2022-09-30 10:54:53 +00:00
|
|
|
|
# Parameters controlling exponential backoff between backfill failures.
|
|
|
|
|
# After the first failure to backfill, we wait 2 hours before trying again. If the
|
|
|
|
|
# second attempt fails, we wait 4 hours before trying again. If the third attempt fails,
|
|
|
|
|
# we wait 8 hours before trying again, ... and so on.
|
|
|
|
|
#
|
|
|
|
|
# Each successive backoff period is twice as long as the last. However we cap this
|
|
|
|
|
# period at a maximum of 2^8 = 256 hours: a little over 10 days. (This is the smallest
|
|
|
|
|
# power of 2 which yields a maximum backoff period of at least 7 days---which was the
|
|
|
|
|
# original maximum backoff period.) Even when we hit this cap, we will continue to
|
|
|
|
|
# make backfill attempts once every 10 days.
|
|
|
|
|
BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS = 8
|
|
|
|
|
BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS = int(
|
|
|
|
|
datetime.timedelta(hours=1).total_seconds() * 1000
|
2022-09-23 19:01:29 +00:00
|
|
|
|
)
|
|
|
|
|
|
2022-09-30 10:54:53 +00:00
|
|
|
|
# We need a cap on the power of 2 or else the backoff period
|
|
|
|
|
# 2^N * (milliseconds per hour)
|
|
|
|
|
# will overflow when calcuated within the database. We ensure overflow does not occur
|
|
|
|
|
# by checking that the largest backoff period fits in a 32-bit signed integer.
|
|
|
|
|
_LONGEST_BACKOFF_PERIOD_MILLISECONDS = (
|
|
|
|
|
2**BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS
|
|
|
|
|
) * BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS
|
|
|
|
|
assert 0 < _LONGEST_BACKOFF_PERIOD_MILLISECONDS <= ((2**31) - 1)
|
|
|
|
|
|
2014-10-28 16:42:35 +00:00
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
# All the info we need while iterating the DAG while backfilling
|
|
|
|
|
@attr.s(frozen=True, slots=True, auto_attribs=True)
|
|
|
|
|
class BackfillQueueNavigationItem:
|
|
|
|
|
depth: int
|
|
|
|
|
stream_ordering: int
|
|
|
|
|
event_id: str
|
|
|
|
|
type: str
|
|
|
|
|
|
|
|
|
|
|
2021-01-11 16:09:22 +00:00
|
|
|
|
class _NoChainCoverIndex(Exception):
|
|
|
|
|
def __init__(self, room_id: str):
|
|
|
|
|
super().__init__("Unexpectedly no chain cover for events in %s" % (room_id,))
|
|
|
|
|
|
|
|
|
|
|
2022-01-21 09:18:10 +00:00
|
|
|
|
class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBaseStore):
|
2023-04-14 18:04:49 +00:00
|
|
|
|
# TODO: this attribute comes from EventPushActionWorkerStore. Should we inherit from
|
|
|
|
|
# that store so that mypy can deduce this for itself?
|
|
|
|
|
stream_ordering_month_ago: Optional[int]
|
|
|
|
|
|
2021-12-13 17:05:00 +00:00
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
database: DatabasePool,
|
|
|
|
|
db_conn: LoggingDatabaseConnection,
|
|
|
|
|
hs: "HomeServer",
|
|
|
|
|
):
|
2020-10-09 11:37:51 +00:00
|
|
|
|
super().__init__(database, db_conn, hs)
|
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
self.hs = hs
|
|
|
|
|
|
2021-09-13 17:07:12 +00:00
|
|
|
|
if hs.config.worker.run_background_tasks:
|
2020-10-09 11:37:51 +00:00
|
|
|
|
hs.get_clock().looping_call(
|
|
|
|
|
self._delete_old_forward_extrem_cache, 60 * 60 * 1000
|
|
|
|
|
)
|
|
|
|
|
|
2020-11-13 11:29:18 +00:00
|
|
|
|
# Cache of event ID to list of auth event IDs and their depths.
|
2021-07-15 16:46:54 +00:00
|
|
|
|
self._event_auth_cache: LruCache[str, List[Tuple[str, int]]] = LruCache(
|
2020-11-13 11:29:18 +00:00
|
|
|
|
500000, "_event_auth_cache", size_callback=len
|
2021-07-15 16:46:54 +00:00
|
|
|
|
)
|
2020-11-13 11:29:18 +00:00
|
|
|
|
|
2021-07-01 09:18:25 +00:00
|
|
|
|
self._clock.looping_call(self._get_stats_for_federation_staging, 30 * 1000)
|
|
|
|
|
|
2023-07-05 09:43:19 +00:00
|
|
|
|
if isinstance(self.database_engine, PostgresEngine):
|
|
|
|
|
self.db_pool.updates.register_background_validate_constraint_and_delete_rows(
|
|
|
|
|
update_name="event_forward_extremities_event_id_foreign_key_constraint_update",
|
|
|
|
|
table="event_forward_extremities",
|
|
|
|
|
constraint_name="event_forward_extremities_event_id",
|
|
|
|
|
constraint=ForeignKeyConstraint("events", [("event_id", "event_id")]),
|
|
|
|
|
unique_columns=("event_id", "room_id"),
|
|
|
|
|
)
|
|
|
|
|
|
2020-08-21 09:06:45 +00:00
|
|
|
|
async def get_auth_chain(
|
2021-03-10 14:57:59 +00:00
|
|
|
|
self, room_id: str, event_ids: Collection[str], include_given: bool = False
|
2020-08-21 09:06:45 +00:00
|
|
|
|
) -> List[EventBase]:
|
2017-05-24 13:22:41 +00:00
|
|
|
|
"""Get auth events for given event_ids. The events *must* be state events.
|
2014-11-07 15:35:53 +00:00
|
|
|
|
|
2017-05-24 13:22:41 +00:00
|
|
|
|
Args:
|
2021-03-10 14:57:59 +00:00
|
|
|
|
room_id: The room the event is in.
|
2020-08-21 09:06:45 +00:00
|
|
|
|
event_ids: state events
|
|
|
|
|
include_given: include the given events in result
|
2017-05-24 13:22:41 +00:00
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
list of events
|
|
|
|
|
"""
|
2020-08-18 20:20:49 +00:00
|
|
|
|
event_ids = await self.get_auth_chain_ids(
|
2021-03-10 14:57:59 +00:00
|
|
|
|
room_id, event_ids, include_given=include_given
|
2020-08-18 20:20:49 +00:00
|
|
|
|
)
|
|
|
|
|
return await self.get_events_as_list(event_ids)
|
2017-05-24 13:22:41 +00:00
|
|
|
|
|
2022-08-15 18:41:23 +00:00
|
|
|
|
@trace
|
|
|
|
|
@tag_args
|
2020-08-21 09:06:45 +00:00
|
|
|
|
async def get_auth_chain_ids(
|
|
|
|
|
self,
|
2021-03-10 14:57:59 +00:00
|
|
|
|
room_id: str,
|
2020-08-21 09:06:45 +00:00
|
|
|
|
event_ids: Collection[str],
|
|
|
|
|
include_given: bool = False,
|
2022-02-12 10:44:16 +00:00
|
|
|
|
) -> Set[str]:
|
2017-05-24 13:22:41 +00:00
|
|
|
|
"""Get auth events for given event_ids. The events *must* be state events.
|
|
|
|
|
|
|
|
|
|
Args:
|
2021-03-10 14:57:59 +00:00
|
|
|
|
room_id: The room the event is in.
|
2020-02-19 15:04:47 +00:00
|
|
|
|
event_ids: state events
|
|
|
|
|
include_given: include the given events in result
|
2017-05-24 13:22:41 +00:00
|
|
|
|
|
|
|
|
|
Returns:
|
2022-02-12 10:44:16 +00:00
|
|
|
|
set of event_ids
|
2017-05-24 13:22:41 +00:00
|
|
|
|
"""
|
2021-03-10 14:57:59 +00:00
|
|
|
|
|
|
|
|
|
# Check if we have indexed the room so we can use the chain cover
|
|
|
|
|
# algorithm.
|
2022-05-18 15:02:10 +00:00
|
|
|
|
room = await self.get_room(room_id) # type: ignore[attr-defined]
|
2021-03-10 14:57:59 +00:00
|
|
|
|
if room["has_auth_chain_index"]:
|
|
|
|
|
try:
|
|
|
|
|
return await self.db_pool.runInteraction(
|
|
|
|
|
"get_auth_chain_ids_chains",
|
|
|
|
|
self._get_auth_chain_ids_using_cover_index_txn,
|
|
|
|
|
room_id,
|
|
|
|
|
event_ids,
|
|
|
|
|
include_given,
|
|
|
|
|
)
|
|
|
|
|
except _NoChainCoverIndex:
|
|
|
|
|
# For whatever reason we don't actually have a chain cover index
|
|
|
|
|
# for the events in question, so we fall back to the old method.
|
|
|
|
|
pass
|
|
|
|
|
|
2020-08-21 09:06:45 +00:00
|
|
|
|
return await self.db_pool.runInteraction(
|
2020-02-19 15:04:47 +00:00
|
|
|
|
"get_auth_chain_ids",
|
|
|
|
|
self._get_auth_chain_ids_txn,
|
|
|
|
|
event_ids,
|
|
|
|
|
include_given,
|
2014-11-07 15:35:53 +00:00
|
|
|
|
)
|
|
|
|
|
|
2021-03-10 14:57:59 +00:00
|
|
|
|
def _get_auth_chain_ids_using_cover_index_txn(
|
2022-05-18 15:02:10 +00:00
|
|
|
|
self,
|
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
|
room_id: str,
|
|
|
|
|
event_ids: Collection[str],
|
|
|
|
|
include_given: bool,
|
2022-02-12 10:44:16 +00:00
|
|
|
|
) -> Set[str]:
|
2021-03-10 14:57:59 +00:00
|
|
|
|
"""Calculates the auth chain IDs using the chain index."""
|
|
|
|
|
|
|
|
|
|
# First we look up the chain ID/sequence numbers for the given events.
|
|
|
|
|
|
|
|
|
|
initial_events = set(event_ids)
|
|
|
|
|
|
|
|
|
|
# All the events that we've found that are reachable from the events.
|
2021-07-15 16:46:54 +00:00
|
|
|
|
seen_events: Set[str] = set()
|
2021-03-10 14:57:59 +00:00
|
|
|
|
|
|
|
|
|
# A map from chain ID to max sequence number of the given events.
|
2021-07-15 16:46:54 +00:00
|
|
|
|
event_chains: Dict[int, int] = {}
|
2021-03-10 14:57:59 +00:00
|
|
|
|
|
|
|
|
|
sql = """
|
|
|
|
|
SELECT event_id, chain_id, sequence_number
|
|
|
|
|
FROM event_auth_chains
|
|
|
|
|
WHERE %s
|
|
|
|
|
"""
|
|
|
|
|
for batch in batch_iter(initial_events, 1000):
|
|
|
|
|
clause, args = make_in_list_sql_clause(
|
|
|
|
|
txn.database_engine, "event_id", batch
|
|
|
|
|
)
|
|
|
|
|
txn.execute(sql % (clause,), args)
|
|
|
|
|
|
|
|
|
|
for event_id, chain_id, sequence_number in txn:
|
|
|
|
|
seen_events.add(event_id)
|
|
|
|
|
event_chains[chain_id] = max(
|
|
|
|
|
sequence_number, event_chains.get(chain_id, 0)
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Check that we actually have a chain ID for all the events.
|
|
|
|
|
events_missing_chain_info = initial_events.difference(seen_events)
|
|
|
|
|
if events_missing_chain_info:
|
|
|
|
|
# This can happen due to e.g. downgrade/upgrade of the server. We
|
|
|
|
|
# raise an exception and fall back to the previous algorithm.
|
|
|
|
|
logger.info(
|
|
|
|
|
"Unexpectedly found that events don't have chain IDs in room %s: %s",
|
|
|
|
|
room_id,
|
|
|
|
|
events_missing_chain_info,
|
|
|
|
|
)
|
|
|
|
|
raise _NoChainCoverIndex(room_id)
|
|
|
|
|
|
|
|
|
|
# Now we look up all links for the chains we have, adding chains that
|
|
|
|
|
# are reachable from any event.
|
|
|
|
|
sql = """
|
|
|
|
|
SELECT
|
|
|
|
|
origin_chain_id, origin_sequence_number,
|
|
|
|
|
target_chain_id, target_sequence_number
|
|
|
|
|
FROM event_auth_chain_links
|
|
|
|
|
WHERE %s
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
# A map from chain ID to max sequence number *reachable* from any event ID.
|
2021-07-15 16:46:54 +00:00
|
|
|
|
chains: Dict[int, int] = {}
|
2021-03-10 14:57:59 +00:00
|
|
|
|
|
|
|
|
|
# Add all linked chains reachable from initial set of chains.
|
2022-05-18 15:02:10 +00:00
|
|
|
|
for batch2 in batch_iter(event_chains, 1000):
|
2021-03-10 14:57:59 +00:00
|
|
|
|
clause, args = make_in_list_sql_clause(
|
2022-05-18 15:02:10 +00:00
|
|
|
|
txn.database_engine, "origin_chain_id", batch2
|
2021-03-10 14:57:59 +00:00
|
|
|
|
)
|
|
|
|
|
txn.execute(sql % (clause,), args)
|
|
|
|
|
|
|
|
|
|
for (
|
|
|
|
|
origin_chain_id,
|
|
|
|
|
origin_sequence_number,
|
|
|
|
|
target_chain_id,
|
|
|
|
|
target_sequence_number,
|
|
|
|
|
) in txn:
|
|
|
|
|
# chains are only reachable if the origin sequence number of
|
|
|
|
|
# the link is less than the max sequence number in the
|
|
|
|
|
# origin chain.
|
|
|
|
|
if origin_sequence_number <= event_chains.get(origin_chain_id, 0):
|
|
|
|
|
chains[target_chain_id] = max(
|
|
|
|
|
target_sequence_number,
|
|
|
|
|
chains.get(target_chain_id, 0),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Add the initial set of chains, excluding the sequence corresponding to
|
|
|
|
|
# initial event.
|
|
|
|
|
for chain_id, seq_no in event_chains.items():
|
|
|
|
|
chains[chain_id] = max(seq_no - 1, chains.get(chain_id, 0))
|
|
|
|
|
|
|
|
|
|
# Now for each chain we figure out the maximum sequence number reachable
|
|
|
|
|
# from *any* event ID. Events with a sequence less than that are in the
|
|
|
|
|
# auth chain.
|
|
|
|
|
if include_given:
|
|
|
|
|
results = initial_events
|
|
|
|
|
else:
|
|
|
|
|
results = set()
|
|
|
|
|
|
|
|
|
|
if isinstance(self.database_engine, PostgresEngine):
|
|
|
|
|
# We can use `execute_values` to efficiently fetch the gaps when
|
|
|
|
|
# using postgres.
|
|
|
|
|
sql = """
|
|
|
|
|
SELECT event_id
|
|
|
|
|
FROM event_auth_chains AS c, (VALUES ?) AS l(chain_id, max_seq)
|
|
|
|
|
WHERE
|
|
|
|
|
c.chain_id = l.chain_id
|
|
|
|
|
AND sequence_number <= max_seq
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
rows = txn.execute_values(sql, chains.items())
|
|
|
|
|
results.update(r for r, in rows)
|
|
|
|
|
else:
|
|
|
|
|
# For SQLite we just fall back to doing a noddy for loop.
|
|
|
|
|
sql = """
|
|
|
|
|
SELECT event_id FROM event_auth_chains
|
|
|
|
|
WHERE chain_id = ? AND sequence_number <= ?
|
|
|
|
|
"""
|
|
|
|
|
for chain_id, max_no in chains.items():
|
|
|
|
|
txn.execute(sql, (chain_id, max_no))
|
|
|
|
|
results.update(r for r, in txn)
|
|
|
|
|
|
2022-02-12 10:44:16 +00:00
|
|
|
|
return results
|
2021-03-10 14:57:59 +00:00
|
|
|
|
|
2020-08-21 09:06:45 +00:00
|
|
|
|
def _get_auth_chain_ids_txn(
|
|
|
|
|
self, txn: LoggingTransaction, event_ids: Collection[str], include_given: bool
|
2022-02-12 10:44:16 +00:00
|
|
|
|
) -> Set[str]:
|
2021-03-10 14:57:59 +00:00
|
|
|
|
"""Calculates the auth chain IDs.
|
|
|
|
|
|
|
|
|
|
This is used when we don't have a cover index for the room.
|
|
|
|
|
"""
|
2020-02-19 09:39:27 +00:00
|
|
|
|
if include_given:
|
|
|
|
|
results = set(event_ids)
|
|
|
|
|
else:
|
|
|
|
|
results = set()
|
|
|
|
|
|
2020-11-13 11:29:18 +00:00
|
|
|
|
# We pull out the depth simply so that we can populate the
|
|
|
|
|
# `_event_auth_cache` cache.
|
|
|
|
|
base_sql = """
|
|
|
|
|
SELECT a.event_id, auth_id, depth
|
|
|
|
|
FROM event_auth AS a
|
|
|
|
|
INNER JOIN events AS e ON (e.event_id = a.auth_id)
|
|
|
|
|
WHERE
|
|
|
|
|
"""
|
2014-11-07 10:42:31 +00:00
|
|
|
|
|
2014-12-16 18:57:36 +00:00
|
|
|
|
front = set(event_ids)
|
2014-11-07 10:42:31 +00:00
|
|
|
|
while front:
|
2022-05-18 15:02:10 +00:00
|
|
|
|
new_front: Set[str] = set()
|
2020-02-19 15:47:11 +00:00
|
|
|
|
for chunk in batch_iter(front, 100):
|
2020-11-13 11:29:18 +00:00
|
|
|
|
# Pull the auth events either from the cache or DB.
|
2021-12-29 13:04:28 +00:00
|
|
|
|
to_fetch: List[str] = [] # Event IDs to fetch from DB
|
2020-11-13 11:29:18 +00:00
|
|
|
|
for event_id in chunk:
|
|
|
|
|
res = self._event_auth_cache.get(event_id)
|
|
|
|
|
if res is None:
|
|
|
|
|
to_fetch.append(event_id)
|
|
|
|
|
else:
|
|
|
|
|
new_front.update(auth_id for auth_id, depth in res)
|
|
|
|
|
|
|
|
|
|
if to_fetch:
|
|
|
|
|
clause, args = make_in_list_sql_clause(
|
|
|
|
|
txn.database_engine, "a.event_id", to_fetch
|
|
|
|
|
)
|
|
|
|
|
txn.execute(base_sql + clause, args)
|
|
|
|
|
|
|
|
|
|
# Note we need to batch up the results by event ID before
|
|
|
|
|
# adding to the cache.
|
2022-05-18 15:02:10 +00:00
|
|
|
|
to_cache: Dict[str, List[Tuple[str, int]]] = {}
|
2020-11-13 11:29:18 +00:00
|
|
|
|
for event_id, auth_event_id, auth_event_depth in txn:
|
|
|
|
|
to_cache.setdefault(event_id, []).append(
|
|
|
|
|
(auth_event_id, auth_event_depth)
|
|
|
|
|
)
|
|
|
|
|
new_front.add(auth_event_id)
|
|
|
|
|
|
|
|
|
|
for event_id, auth_events in to_cache.items():
|
|
|
|
|
self._event_auth_cache.set(event_id, auth_events)
|
2015-02-19 17:24:14 +00:00
|
|
|
|
|
|
|
|
|
new_front -= results
|
|
|
|
|
|
2015-02-12 14:39:31 +00:00
|
|
|
|
front = new_front
|
2014-11-07 11:21:20 +00:00
|
|
|
|
results.update(front)
|
2014-11-07 10:42:31 +00:00
|
|
|
|
|
2022-02-12 10:44:16 +00:00
|
|
|
|
return results
|
2014-11-07 10:42:31 +00:00
|
|
|
|
|
2020-12-04 15:52:49 +00:00
|
|
|
|
async def get_auth_chain_difference(
|
|
|
|
|
self, room_id: str, state_sets: List[Set[str]]
|
|
|
|
|
) -> Set[str]:
|
2020-03-18 16:46:41 +00:00
|
|
|
|
"""Given sets of state events figure out the auth chain difference (as
|
|
|
|
|
per state res v2 algorithm).
|
|
|
|
|
|
|
|
|
|
This equivalent to fetching the full auth chain for each set of state
|
|
|
|
|
and returning the events that don't appear in each and every auth
|
|
|
|
|
chain.
|
|
|
|
|
|
|
|
|
|
Returns:
|
2020-08-28 11:54:27 +00:00
|
|
|
|
The set of the difference in auth chains.
|
2020-03-18 16:46:41 +00:00
|
|
|
|
"""
|
|
|
|
|
|
2021-01-11 16:09:22 +00:00
|
|
|
|
# Check if we have indexed the room so we can use the chain cover
|
|
|
|
|
# algorithm.
|
2022-05-18 15:02:10 +00:00
|
|
|
|
room = await self.get_room(room_id) # type: ignore[attr-defined]
|
2021-01-11 16:09:22 +00:00
|
|
|
|
if room["has_auth_chain_index"]:
|
|
|
|
|
try:
|
|
|
|
|
return await self.db_pool.runInteraction(
|
|
|
|
|
"get_auth_chain_difference_chains",
|
|
|
|
|
self._get_auth_chain_difference_using_cover_index_txn,
|
|
|
|
|
room_id,
|
|
|
|
|
state_sets,
|
|
|
|
|
)
|
|
|
|
|
except _NoChainCoverIndex:
|
|
|
|
|
# For whatever reason we don't actually have a chain cover index
|
|
|
|
|
# for the events in question, so we fall back to the old method.
|
|
|
|
|
pass
|
|
|
|
|
|
2020-08-28 11:54:27 +00:00
|
|
|
|
return await self.db_pool.runInteraction(
|
2020-03-18 16:46:41 +00:00
|
|
|
|
"get_auth_chain_difference",
|
|
|
|
|
self._get_auth_chain_difference_txn,
|
|
|
|
|
state_sets,
|
|
|
|
|
)
|
|
|
|
|
|
2021-01-11 16:09:22 +00:00
|
|
|
|
def _get_auth_chain_difference_using_cover_index_txn(
|
2022-05-18 15:02:10 +00:00
|
|
|
|
self, txn: LoggingTransaction, room_id: str, state_sets: List[Set[str]]
|
2021-01-11 16:09:22 +00:00
|
|
|
|
) -> Set[str]:
|
|
|
|
|
"""Calculates the auth chain difference using the chain index.
|
|
|
|
|
|
|
|
|
|
See docs/auth_chain_difference_algorithm.md for details
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
# First we look up the chain ID/sequence numbers for all the events, and
|
|
|
|
|
# work out the chain/sequence numbers reachable from each state set.
|
|
|
|
|
|
|
|
|
|
initial_events = set(state_sets[0]).union(*state_sets[1:])
|
|
|
|
|
|
|
|
|
|
# Map from event_id -> (chain ID, seq no)
|
2021-07-15 16:46:54 +00:00
|
|
|
|
chain_info: Dict[str, Tuple[int, int]] = {}
|
2021-01-11 16:09:22 +00:00
|
|
|
|
|
|
|
|
|
# Map from chain ID -> seq no -> event Id
|
2021-07-15 16:46:54 +00:00
|
|
|
|
chain_to_event: Dict[int, Dict[int, str]] = {}
|
2021-01-11 16:09:22 +00:00
|
|
|
|
|
|
|
|
|
# All the chains that we've found that are reachable from the state
|
|
|
|
|
# sets.
|
2021-07-15 16:46:54 +00:00
|
|
|
|
seen_chains: Set[int] = set()
|
2021-01-11 16:09:22 +00:00
|
|
|
|
|
|
|
|
|
sql = """
|
|
|
|
|
SELECT event_id, chain_id, sequence_number
|
|
|
|
|
FROM event_auth_chains
|
|
|
|
|
WHERE %s
|
|
|
|
|
"""
|
|
|
|
|
for batch in batch_iter(initial_events, 1000):
|
|
|
|
|
clause, args = make_in_list_sql_clause(
|
|
|
|
|
txn.database_engine, "event_id", batch
|
|
|
|
|
)
|
|
|
|
|
txn.execute(sql % (clause,), args)
|
|
|
|
|
|
|
|
|
|
for event_id, chain_id, sequence_number in txn:
|
|
|
|
|
chain_info[event_id] = (chain_id, sequence_number)
|
|
|
|
|
seen_chains.add(chain_id)
|
|
|
|
|
chain_to_event.setdefault(chain_id, {})[sequence_number] = event_id
|
|
|
|
|
|
|
|
|
|
# Check that we actually have a chain ID for all the events.
|
|
|
|
|
events_missing_chain_info = initial_events.difference(chain_info)
|
|
|
|
|
if events_missing_chain_info:
|
|
|
|
|
# This can happen due to e.g. downgrade/upgrade of the server. We
|
|
|
|
|
# raise an exception and fall back to the previous algorithm.
|
|
|
|
|
logger.info(
|
|
|
|
|
"Unexpectedly found that events don't have chain IDs in room %s: %s",
|
|
|
|
|
room_id,
|
|
|
|
|
events_missing_chain_info,
|
|
|
|
|
)
|
|
|
|
|
raise _NoChainCoverIndex(room_id)
|
|
|
|
|
|
|
|
|
|
# Corresponds to `state_sets`, except as a map from chain ID to max
|
|
|
|
|
# sequence number reachable from the state set.
|
2021-07-15 16:46:54 +00:00
|
|
|
|
set_to_chain: List[Dict[int, int]] = []
|
2021-01-11 16:09:22 +00:00
|
|
|
|
for state_set in state_sets:
|
2021-07-15 16:46:54 +00:00
|
|
|
|
chains: Dict[int, int] = {}
|
2021-01-11 16:09:22 +00:00
|
|
|
|
set_to_chain.append(chains)
|
|
|
|
|
|
|
|
|
|
for event_id in state_set:
|
|
|
|
|
chain_id, seq_no = chain_info[event_id]
|
|
|
|
|
|
|
|
|
|
chains[chain_id] = max(seq_no, chains.get(chain_id, 0))
|
|
|
|
|
|
|
|
|
|
# Now we look up all links for the chains we have, adding chains to
|
|
|
|
|
# set_to_chain that are reachable from each set.
|
|
|
|
|
sql = """
|
|
|
|
|
SELECT
|
|
|
|
|
origin_chain_id, origin_sequence_number,
|
|
|
|
|
target_chain_id, target_sequence_number
|
|
|
|
|
FROM event_auth_chain_links
|
|
|
|
|
WHERE %s
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
# (We need to take a copy of `seen_chains` as we want to mutate it in
|
|
|
|
|
# the loop)
|
2022-05-18 15:02:10 +00:00
|
|
|
|
for batch2 in batch_iter(set(seen_chains), 1000):
|
2021-01-11 16:09:22 +00:00
|
|
|
|
clause, args = make_in_list_sql_clause(
|
2022-05-18 15:02:10 +00:00
|
|
|
|
txn.database_engine, "origin_chain_id", batch2
|
2021-01-11 16:09:22 +00:00
|
|
|
|
)
|
|
|
|
|
txn.execute(sql % (clause,), args)
|
|
|
|
|
|
|
|
|
|
for (
|
|
|
|
|
origin_chain_id,
|
|
|
|
|
origin_sequence_number,
|
|
|
|
|
target_chain_id,
|
|
|
|
|
target_sequence_number,
|
|
|
|
|
) in txn:
|
|
|
|
|
for chains in set_to_chain:
|
|
|
|
|
# chains are only reachable if the origin sequence number of
|
|
|
|
|
# the link is less than the max sequence number in the
|
|
|
|
|
# origin chain.
|
|
|
|
|
if origin_sequence_number <= chains.get(origin_chain_id, 0):
|
|
|
|
|
chains[target_chain_id] = max(
|
|
|
|
|
target_sequence_number,
|
|
|
|
|
chains.get(target_chain_id, 0),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
seen_chains.add(target_chain_id)
|
|
|
|
|
|
|
|
|
|
# Now for each chain we figure out the maximum sequence number reachable
|
|
|
|
|
# from *any* state set and the minimum sequence number reachable from
|
|
|
|
|
# *all* state sets. Events in that range are in the auth chain
|
|
|
|
|
# difference.
|
|
|
|
|
result = set()
|
|
|
|
|
|
|
|
|
|
# Mapping from chain ID to the range of sequence numbers that should be
|
|
|
|
|
# pulled from the database.
|
2021-07-15 16:46:54 +00:00
|
|
|
|
chain_to_gap: Dict[int, Tuple[int, int]] = {}
|
2021-01-11 16:09:22 +00:00
|
|
|
|
|
|
|
|
|
for chain_id in seen_chains:
|
|
|
|
|
min_seq_no = min(chains.get(chain_id, 0) for chains in set_to_chain)
|
|
|
|
|
max_seq_no = max(chains.get(chain_id, 0) for chains in set_to_chain)
|
|
|
|
|
|
|
|
|
|
if min_seq_no < max_seq_no:
|
|
|
|
|
# We have a non empty gap, try and fill it from the events that
|
|
|
|
|
# we have, otherwise add them to the list of gaps to pull out
|
|
|
|
|
# from the DB.
|
|
|
|
|
for seq_no in range(min_seq_no + 1, max_seq_no + 1):
|
|
|
|
|
event_id = chain_to_event.get(chain_id, {}).get(seq_no)
|
|
|
|
|
if event_id:
|
|
|
|
|
result.add(event_id)
|
|
|
|
|
else:
|
|
|
|
|
chain_to_gap[chain_id] = (min_seq_no, max_seq_no)
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
if not chain_to_gap:
|
|
|
|
|
# If there are no gaps to fetch, we're done!
|
|
|
|
|
return result
|
|
|
|
|
|
|
|
|
|
if isinstance(self.database_engine, PostgresEngine):
|
|
|
|
|
# We can use `execute_values` to efficiently fetch the gaps when
|
|
|
|
|
# using postgres.
|
|
|
|
|
sql = """
|
|
|
|
|
SELECT event_id
|
|
|
|
|
FROM event_auth_chains AS c, (VALUES ?) AS l(chain_id, min_seq, max_seq)
|
|
|
|
|
WHERE
|
|
|
|
|
c.chain_id = l.chain_id
|
|
|
|
|
AND min_seq < sequence_number AND sequence_number <= max_seq
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
args = [
|
|
|
|
|
(chain_id, min_no, max_no)
|
|
|
|
|
for chain_id, (min_no, max_no) in chain_to_gap.items()
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
rows = txn.execute_values(sql, args)
|
|
|
|
|
result.update(r for r, in rows)
|
|
|
|
|
else:
|
|
|
|
|
# For SQLite we just fall back to doing a noddy for loop.
|
|
|
|
|
sql = """
|
|
|
|
|
SELECT event_id FROM event_auth_chains
|
|
|
|
|
WHERE chain_id = ? AND ? < sequence_number AND sequence_number <= ?
|
|
|
|
|
"""
|
|
|
|
|
for chain_id, (min_no, max_no) in chain_to_gap.items():
|
|
|
|
|
txn.execute(sql, (chain_id, min_no, max_no))
|
|
|
|
|
result.update(r for r, in txn)
|
|
|
|
|
|
|
|
|
|
return result
|
|
|
|
|
|
2020-03-18 16:46:41 +00:00
|
|
|
|
def _get_auth_chain_difference_txn(
|
2022-05-18 15:02:10 +00:00
|
|
|
|
self, txn: LoggingTransaction, state_sets: List[Set[str]]
|
2020-03-18 16:46:41 +00:00
|
|
|
|
) -> Set[str]:
|
2021-01-11 16:09:22 +00:00
|
|
|
|
"""Calculates the auth chain difference using a breadth first search.
|
|
|
|
|
|
|
|
|
|
This is used when we don't have a cover index for the room.
|
|
|
|
|
"""
|
2020-03-18 16:46:41 +00:00
|
|
|
|
|
|
|
|
|
# Algorithm Description
|
|
|
|
|
# ~~~~~~~~~~~~~~~~~~~~~
|
|
|
|
|
#
|
|
|
|
|
# The idea here is to basically walk the auth graph of each state set in
|
|
|
|
|
# tandem, keeping track of which auth events are reachable by each state
|
|
|
|
|
# set. If we reach an auth event we've already visited (via a different
|
|
|
|
|
# state set) then we mark that auth event and all ancestors as reachable
|
|
|
|
|
# by the state set. This requires that we keep track of the auth chains
|
|
|
|
|
# in memory.
|
|
|
|
|
#
|
|
|
|
|
# Doing it in a such a way means that we can stop early if all auth
|
|
|
|
|
# events we're currently walking are reachable by all state sets.
|
|
|
|
|
#
|
|
|
|
|
# *Note*: We can't stop walking an event's auth chain if it is reachable
|
|
|
|
|
# by all state sets. This is because other auth chains we're walking
|
|
|
|
|
# might be reachable only via the original auth chain. For example,
|
|
|
|
|
# given the following auth chain:
|
|
|
|
|
#
|
|
|
|
|
# A -> C -> D -> E
|
|
|
|
|
# / /
|
|
|
|
|
# B -´---------´
|
|
|
|
|
#
|
|
|
|
|
# and state sets {A} and {B} then walking the auth chains of A and B
|
|
|
|
|
# would immediately show that C is reachable by both. However, if we
|
|
|
|
|
# stopped at C then we'd only reach E via the auth chain of B and so E
|
2021-02-12 16:01:48 +00:00
|
|
|
|
# would erroneously get included in the returned difference.
|
2020-03-18 16:46:41 +00:00
|
|
|
|
#
|
|
|
|
|
# The other thing that we do is limit the number of auth chains we walk
|
|
|
|
|
# at once, due to practical limits (i.e. we can only query the database
|
|
|
|
|
# with a limited set of parameters). We pick the auth chains we walk
|
|
|
|
|
# each iteration based on their depth, in the hope that events with a
|
|
|
|
|
# lower depth are likely reachable by those with higher depths.
|
|
|
|
|
#
|
|
|
|
|
# We could use any ordering that we believe would give a rough
|
|
|
|
|
# topological ordering, e.g. origin server timestamp. If the ordering
|
|
|
|
|
# chosen is not topological then the algorithm still produces the right
|
|
|
|
|
# result, but perhaps a bit more inefficiently. This is why it is safe
|
|
|
|
|
# to use "depth" here.
|
|
|
|
|
|
|
|
|
|
initial_events = set(state_sets[0]).union(*state_sets[1:])
|
|
|
|
|
|
|
|
|
|
# Dict from events in auth chains to which sets *cannot* reach them.
|
|
|
|
|
# I.e. if the set is empty then all sets can reach the event.
|
|
|
|
|
event_to_missing_sets = {
|
|
|
|
|
event_id: {i for i, a in enumerate(state_sets) if event_id not in a}
|
|
|
|
|
for event_id in initial_events
|
|
|
|
|
}
|
|
|
|
|
|
2020-04-15 09:16:35 +00:00
|
|
|
|
# The sorted list of events whose auth chains we should walk.
|
2021-07-15 16:46:54 +00:00
|
|
|
|
search: List[Tuple[int, str]] = []
|
2020-04-15 09:16:35 +00:00
|
|
|
|
|
2020-03-18 16:46:41 +00:00
|
|
|
|
# We need to get the depth of the initial events for sorting purposes.
|
|
|
|
|
sql = """
|
|
|
|
|
SELECT depth, event_id FROM events
|
|
|
|
|
WHERE %s
|
|
|
|
|
"""
|
2020-04-15 09:16:35 +00:00
|
|
|
|
# the list can be huge, so let's avoid looking them all up in one massive
|
|
|
|
|
# query.
|
|
|
|
|
for batch in batch_iter(initial_events, 1000):
|
|
|
|
|
clause, args = make_in_list_sql_clause(
|
|
|
|
|
txn.database_engine, "event_id", batch
|
|
|
|
|
)
|
|
|
|
|
txn.execute(sql % (clause,), args)
|
|
|
|
|
|
|
|
|
|
# I think building a temporary list with fetchall is more efficient than
|
|
|
|
|
# just `search.extend(txn)`, but this is unconfirmed
|
2022-05-18 15:02:10 +00:00
|
|
|
|
search.extend(cast(List[Tuple[int, str]], txn.fetchall()))
|
2020-04-15 09:16:35 +00:00
|
|
|
|
|
|
|
|
|
# sort by depth
|
|
|
|
|
search.sort()
|
2020-03-18 16:46:41 +00:00
|
|
|
|
|
|
|
|
|
# Map from event to its auth events
|
2021-07-15 16:46:54 +00:00
|
|
|
|
event_to_auth_events: Dict[str, Set[str]] = {}
|
2020-03-18 16:46:41 +00:00
|
|
|
|
|
|
|
|
|
base_sql = """
|
|
|
|
|
SELECT a.event_id, auth_id, depth
|
|
|
|
|
FROM event_auth AS a
|
|
|
|
|
INNER JOIN events AS e ON (e.event_id = a.auth_id)
|
|
|
|
|
WHERE
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
while search:
|
|
|
|
|
# Check whether all our current walks are reachable by all state
|
|
|
|
|
# sets. If so we can bail.
|
|
|
|
|
if all(not event_to_missing_sets[eid] for _, eid in search):
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
# Fetch the auth events and their depths of the N last events we're
|
2020-11-13 11:29:18 +00:00
|
|
|
|
# currently walking, either from cache or DB.
|
2020-03-18 16:46:41 +00:00
|
|
|
|
search, chunk = search[:-100], search[-100:]
|
|
|
|
|
|
2021-12-29 13:04:28 +00:00
|
|
|
|
found: List[Tuple[str, str, int]] = [] # Results found
|
|
|
|
|
to_fetch: List[str] = [] # Event IDs to fetch from DB
|
2020-11-13 11:29:18 +00:00
|
|
|
|
for _, event_id in chunk:
|
|
|
|
|
res = self._event_auth_cache.get(event_id)
|
|
|
|
|
if res is None:
|
|
|
|
|
to_fetch.append(event_id)
|
|
|
|
|
else:
|
|
|
|
|
found.extend((event_id, auth_id, depth) for auth_id, depth in res)
|
|
|
|
|
|
|
|
|
|
if to_fetch:
|
|
|
|
|
clause, args = make_in_list_sql_clause(
|
|
|
|
|
txn.database_engine, "a.event_id", to_fetch
|
|
|
|
|
)
|
|
|
|
|
txn.execute(base_sql + clause, args)
|
|
|
|
|
|
|
|
|
|
# We parse the results and add the to the `found` set and the
|
|
|
|
|
# cache (note we need to batch up the results by event ID before
|
|
|
|
|
# adding to the cache).
|
2022-05-18 15:02:10 +00:00
|
|
|
|
to_cache: Dict[str, List[Tuple[str, int]]] = {}
|
2020-11-13 11:29:18 +00:00
|
|
|
|
for event_id, auth_event_id, auth_event_depth in txn:
|
|
|
|
|
to_cache.setdefault(event_id, []).append(
|
|
|
|
|
(auth_event_id, auth_event_depth)
|
|
|
|
|
)
|
|
|
|
|
found.append((event_id, auth_event_id, auth_event_depth))
|
|
|
|
|
|
|
|
|
|
for event_id, auth_events in to_cache.items():
|
|
|
|
|
self._event_auth_cache.set(event_id, auth_events)
|
|
|
|
|
|
|
|
|
|
for event_id, auth_event_id, auth_event_depth in found:
|
2020-03-18 16:46:41 +00:00
|
|
|
|
event_to_auth_events.setdefault(event_id, set()).add(auth_event_id)
|
|
|
|
|
|
|
|
|
|
sets = event_to_missing_sets.get(auth_event_id)
|
|
|
|
|
if sets is None:
|
|
|
|
|
# First time we're seeing this event, so we add it to the
|
|
|
|
|
# queue of things to fetch.
|
|
|
|
|
search.append((auth_event_depth, auth_event_id))
|
|
|
|
|
|
|
|
|
|
# Assume that this event is unreachable from any of the
|
|
|
|
|
# state sets until proven otherwise
|
|
|
|
|
sets = event_to_missing_sets[auth_event_id] = set(
|
|
|
|
|
range(len(state_sets))
|
|
|
|
|
)
|
|
|
|
|
else:
|
|
|
|
|
# We've previously seen this event, so look up its auth
|
|
|
|
|
# events and recursively mark all ancestors as reachable
|
|
|
|
|
# by the current event's state set.
|
|
|
|
|
a_ids = event_to_auth_events.get(auth_event_id)
|
|
|
|
|
while a_ids:
|
|
|
|
|
new_aids = set()
|
|
|
|
|
for a_id in a_ids:
|
|
|
|
|
event_to_missing_sets[a_id].intersection_update(
|
|
|
|
|
event_to_missing_sets[event_id]
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
b = event_to_auth_events.get(a_id)
|
|
|
|
|
if b:
|
|
|
|
|
new_aids.update(b)
|
|
|
|
|
|
|
|
|
|
a_ids = new_aids
|
|
|
|
|
|
2021-02-12 16:01:48 +00:00
|
|
|
|
# Mark that the auth event is reachable by the appropriate sets.
|
2020-03-18 16:46:41 +00:00
|
|
|
|
sets.intersection_update(event_to_missing_sets[event_id])
|
|
|
|
|
|
|
|
|
|
search.sort()
|
|
|
|
|
|
|
|
|
|
# Return all events where not all sets can reach them.
|
|
|
|
|
return {eid for eid, n in event_to_missing_sets.items() if n}
|
|
|
|
|
|
2022-08-16 17:39:40 +00:00
|
|
|
|
@trace
|
|
|
|
|
@tag_args
|
2022-09-23 19:01:29 +00:00
|
|
|
|
async def get_backfill_points_in_room(
|
|
|
|
|
self,
|
|
|
|
|
room_id: str,
|
2022-09-28 20:26:16 +00:00
|
|
|
|
current_depth: int,
|
|
|
|
|
limit: int,
|
2022-04-26 09:27:11 +00:00
|
|
|
|
) -> List[Tuple[str, int]]:
|
2022-09-23 19:01:29 +00:00
|
|
|
|
"""
|
2022-09-28 20:26:16 +00:00
|
|
|
|
Get the backward extremities to backfill from in the room along with the
|
|
|
|
|
approximate depth.
|
|
|
|
|
|
|
|
|
|
Only returns events that are at a depth lower than or
|
|
|
|
|
equal to the `current_depth`. Sorted by depth, highest to lowest (descending)
|
|
|
|
|
so the closest events to the `current_depth` are first in the list.
|
|
|
|
|
|
|
|
|
|
We ignore extremities that are newer than the user's current scroll position
|
|
|
|
|
(ie, those with depth greater than `current_depth`) as:
|
|
|
|
|
1. we don't really care about getting events that have happened
|
|
|
|
|
after our current position; and
|
|
|
|
|
2. by the nature of paginating and scrolling back, we have likely
|
|
|
|
|
previously tried and failed to backfill from that extremity, so
|
|
|
|
|
to avoid getting "stuck" requesting the same backfill repeatedly
|
|
|
|
|
we drop those extremities.
|
Add support for MSC2716 marker events (#10498)
* Make historical messages available to federated servers
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
Follow-up to https://github.com/matrix-org/synapse/pull/9247
* Debug message not available on federation
* Add base starting insertion point when no chunk ID is provided
* Fix messages from multiple senders in historical chunk
Follow-up to https://github.com/matrix-org/synapse/pull/9247
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
---
Previously, Synapse would throw a 403,
`Cannot force another user to join.`,
because we were trying to use `?user_id` from a single virtual user
which did not match with messages from other users in the chunk.
* Remove debug lines
* Messing with selecting insertion event extremeties
* Move db schema change to new version
* Add more better comments
* Make a fake requester with just what we need
See https://github.com/matrix-org/synapse/pull/10276#discussion_r660999080
* Store insertion events in table
* Make base insertion event float off on its own
See https://github.com/matrix-org/synapse/pull/10250#issuecomment-875711889
Conflicts:
synapse/rest/client/v1/room.py
* Validate that the app service can actually control the given user
See https://github.com/matrix-org/synapse/pull/10276#issuecomment-876316455
Conflicts:
synapse/rest/client/v1/room.py
* Add some better comments on what we're trying to check for
* Continue debugging
* Share validation logic
* Add inserted historical messages to /backfill response
* Remove debug sql queries
* Some marker event implemntation trials
* Clean up PR
* Rename insertion_event_id to just event_id
* Add some better sql comments
* More accurate description
* Add changelog
* Make it clear what MSC the change is part of
* Add more detail on which insertion event came through
* Address review and improve sql queries
* Only use event_id as unique constraint
* Fix test case where insertion event is already in the normal DAG
* Remove debug changes
* Add support for MSC2716 marker events
* Process markers when we receive it over federation
* WIP: make hs2 backfill historical messages after marker event
* hs2 to better ask for insertion event extremity
But running into the `sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group`
error
* Add insertion_event_extremities table
* Switch to chunk events so we can auth via power_levels
Previously, we were using `content.chunk_id` to connect one
chunk to another. But these events can be from any `sender`
and we can't tell who should be able to send historical events.
We know we only want the application service to do it but these
events have the sender of a real historical message, not the
application service user ID as the sender. Other federated homeservers
also have no indicator which senders are an application service on
the originating homeserver.
So we want to auth all of the MSC2716 events via power_levels
and have them be sent by the application service with proper
PL levels in the room.
* Switch to chunk events for federation
* Add unstable room version to support new historical PL
* Messy: Fix undefined state_group for federated historical events
```
2021-07-13 02:27:57,810 - synapse.handlers.federation - 1248 - ERROR - GET-4 - Failed to backfill from hs1 because NOT NULL constraint failed: event_to_state_groups.state_group
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1216, in try_backfill
await self.backfill(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1035, in backfill
await self._auth_and_persist_event(dest, event, context, backfilled=True)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2222, in _auth_and_persist_event
await self._run_push_actions_and_persist_event(event, context, backfilled)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2244, in _run_push_actions_and_persist_event
await self.persist_events_and_notify(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 3290, in persist_events_and_notify
events, max_stream_token = await self.storage.persistence.persist_events(
File "/usr/local/lib/python3.8/site-packages/synapse/logging/opentracing.py", line 774, in _trace_inner
return await func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 320, in persist_events
ret_vals = await yieldable_gather_results(enqueue, partitioned.items())
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 237, in handle_queue_loop
ret = await self._per_item_callback(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 577, in _persist_event_batch
await self.persist_events_store._persist_events_and_state_updates(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 176, in _persist_events_and_state_updates
await self.db_pool.runInteraction(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 681, in runInteraction
result = await self.runWithConnection(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 770, in runWithConnection
return await make_deferred_yieldable(
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 238, in inContext
result = inContext.theWork() # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 254, in <lambda>
inContext.theWork = lambda: context.call( # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 83, in callWithContext
return func(*args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 293, in _runWithConnection
compat.reraise(excValue, excTraceback)
File "/usr/local/lib/python3.8/site-packages/twisted/python/deprecate.py", line 298, in deprecatedFunction
return function(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/twisted/python/compat.py", line 403, in reraise
raise exception.with_traceback(traceback)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 284, in _runWithConnection
result = func(conn, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 765, in inner_func
return func(db_conn, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 549, in new_transaction
r = func(cursor, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/logging/utils.py", line 69, in wrapped
return f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 385, in _persist_events_txn
self._store_event_state_mappings_txn(txn, events_and_contexts)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 2065, in _store_event_state_mappings_txn
self.db_pool.simple_insert_many_txn(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 923, in simple_insert_many_txn
txn.execute_batch(sql, vals)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 280, in execute_batch
self.executemany(sql, args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 300, in executemany
self._do_execute(self.txn.executemany, sql, *args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 330, in _do_execute
return func(sql, *args)
sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group
```
* Revert "Messy: Fix undefined state_group for federated historical events"
This reverts commit 187ab28611546321e02770944c86f30ee2bc742a.
* Fix federated events being rejected for no state_groups
Add fix from https://github.com/matrix-org/synapse/pull/10439
until it merges.
* Adapting to experimental room version
* Some log cleanup
* Add better comments around extremity fetching code and why
* Rename to be more accurate to what the function returns
* Add changelog
* Ignore rejected events
* Use simplified upsert
* Add Erik's explanation of extra event checks
See https://github.com/matrix-org/synapse/pull/10498#discussion_r680880332
* Clarify that the depth is not directly correlated to the backwards extremity that we return
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681725404
* lock only matters for sqlite
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681728061
* Move new SQL changes to its own delta file
* Clean up upsert docstring
* Bump database schema version (62)
2021-08-04 17:07:57 +00:00
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
room_id: Room where we want to find the oldest events
|
2022-09-28 20:26:16 +00:00
|
|
|
|
current_depth: The depth at the user's current scrollback position
|
|
|
|
|
limit: The max number of backfill points to return
|
Add support for MSC2716 marker events (#10498)
* Make historical messages available to federated servers
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
Follow-up to https://github.com/matrix-org/synapse/pull/9247
* Debug message not available on federation
* Add base starting insertion point when no chunk ID is provided
* Fix messages from multiple senders in historical chunk
Follow-up to https://github.com/matrix-org/synapse/pull/9247
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
---
Previously, Synapse would throw a 403,
`Cannot force another user to join.`,
because we were trying to use `?user_id` from a single virtual user
which did not match with messages from other users in the chunk.
* Remove debug lines
* Messing with selecting insertion event extremeties
* Move db schema change to new version
* Add more better comments
* Make a fake requester with just what we need
See https://github.com/matrix-org/synapse/pull/10276#discussion_r660999080
* Store insertion events in table
* Make base insertion event float off on its own
See https://github.com/matrix-org/synapse/pull/10250#issuecomment-875711889
Conflicts:
synapse/rest/client/v1/room.py
* Validate that the app service can actually control the given user
See https://github.com/matrix-org/synapse/pull/10276#issuecomment-876316455
Conflicts:
synapse/rest/client/v1/room.py
* Add some better comments on what we're trying to check for
* Continue debugging
* Share validation logic
* Add inserted historical messages to /backfill response
* Remove debug sql queries
* Some marker event implemntation trials
* Clean up PR
* Rename insertion_event_id to just event_id
* Add some better sql comments
* More accurate description
* Add changelog
* Make it clear what MSC the change is part of
* Add more detail on which insertion event came through
* Address review and improve sql queries
* Only use event_id as unique constraint
* Fix test case where insertion event is already in the normal DAG
* Remove debug changes
* Add support for MSC2716 marker events
* Process markers when we receive it over federation
* WIP: make hs2 backfill historical messages after marker event
* hs2 to better ask for insertion event extremity
But running into the `sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group`
error
* Add insertion_event_extremities table
* Switch to chunk events so we can auth via power_levels
Previously, we were using `content.chunk_id` to connect one
chunk to another. But these events can be from any `sender`
and we can't tell who should be able to send historical events.
We know we only want the application service to do it but these
events have the sender of a real historical message, not the
application service user ID as the sender. Other federated homeservers
also have no indicator which senders are an application service on
the originating homeserver.
So we want to auth all of the MSC2716 events via power_levels
and have them be sent by the application service with proper
PL levels in the room.
* Switch to chunk events for federation
* Add unstable room version to support new historical PL
* Messy: Fix undefined state_group for federated historical events
```
2021-07-13 02:27:57,810 - synapse.handlers.federation - 1248 - ERROR - GET-4 - Failed to backfill from hs1 because NOT NULL constraint failed: event_to_state_groups.state_group
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1216, in try_backfill
await self.backfill(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1035, in backfill
await self._auth_and_persist_event(dest, event, context, backfilled=True)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2222, in _auth_and_persist_event
await self._run_push_actions_and_persist_event(event, context, backfilled)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2244, in _run_push_actions_and_persist_event
await self.persist_events_and_notify(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 3290, in persist_events_and_notify
events, max_stream_token = await self.storage.persistence.persist_events(
File "/usr/local/lib/python3.8/site-packages/synapse/logging/opentracing.py", line 774, in _trace_inner
return await func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 320, in persist_events
ret_vals = await yieldable_gather_results(enqueue, partitioned.items())
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 237, in handle_queue_loop
ret = await self._per_item_callback(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 577, in _persist_event_batch
await self.persist_events_store._persist_events_and_state_updates(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 176, in _persist_events_and_state_updates
await self.db_pool.runInteraction(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 681, in runInteraction
result = await self.runWithConnection(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 770, in runWithConnection
return await make_deferred_yieldable(
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 238, in inContext
result = inContext.theWork() # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 254, in <lambda>
inContext.theWork = lambda: context.call( # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 83, in callWithContext
return func(*args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 293, in _runWithConnection
compat.reraise(excValue, excTraceback)
File "/usr/local/lib/python3.8/site-packages/twisted/python/deprecate.py", line 298, in deprecatedFunction
return function(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/twisted/python/compat.py", line 403, in reraise
raise exception.with_traceback(traceback)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 284, in _runWithConnection
result = func(conn, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 765, in inner_func
return func(db_conn, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 549, in new_transaction
r = func(cursor, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/logging/utils.py", line 69, in wrapped
return f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 385, in _persist_events_txn
self._store_event_state_mappings_txn(txn, events_and_contexts)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 2065, in _store_event_state_mappings_txn
self.db_pool.simple_insert_many_txn(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 923, in simple_insert_many_txn
txn.execute_batch(sql, vals)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 280, in execute_batch
self.executemany(sql, args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 300, in executemany
self._do_execute(self.txn.executemany, sql, *args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 330, in _do_execute
return func(sql, *args)
sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group
```
* Revert "Messy: Fix undefined state_group for federated historical events"
This reverts commit 187ab28611546321e02770944c86f30ee2bc742a.
* Fix federated events being rejected for no state_groups
Add fix from https://github.com/matrix-org/synapse/pull/10439
until it merges.
* Adapting to experimental room version
* Some log cleanup
* Add better comments around extremity fetching code and why
* Rename to be more accurate to what the function returns
* Add changelog
* Ignore rejected events
* Use simplified upsert
* Add Erik's explanation of extra event checks
See https://github.com/matrix-org/synapse/pull/10498#discussion_r680880332
* Clarify that the depth is not directly correlated to the backwards extremity that we return
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681725404
* lock only matters for sqlite
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681728061
* Move new SQL changes to its own delta file
* Clean up upsert docstring
* Bump database schema version (62)
2021-08-04 17:07:57 +00:00
|
|
|
|
|
|
|
|
|
Returns:
|
2022-09-23 19:01:29 +00:00
|
|
|
|
List of (event_id, depth) tuples. Sorted by depth, highest to lowest
|
2022-09-28 20:26:16 +00:00
|
|
|
|
(descending) so the closest events to the `current_depth` are first
|
|
|
|
|
in the list.
|
Add support for MSC2716 marker events (#10498)
* Make historical messages available to federated servers
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
Follow-up to https://github.com/matrix-org/synapse/pull/9247
* Debug message not available on federation
* Add base starting insertion point when no chunk ID is provided
* Fix messages from multiple senders in historical chunk
Follow-up to https://github.com/matrix-org/synapse/pull/9247
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
---
Previously, Synapse would throw a 403,
`Cannot force another user to join.`,
because we were trying to use `?user_id` from a single virtual user
which did not match with messages from other users in the chunk.
* Remove debug lines
* Messing with selecting insertion event extremeties
* Move db schema change to new version
* Add more better comments
* Make a fake requester with just what we need
See https://github.com/matrix-org/synapse/pull/10276#discussion_r660999080
* Store insertion events in table
* Make base insertion event float off on its own
See https://github.com/matrix-org/synapse/pull/10250#issuecomment-875711889
Conflicts:
synapse/rest/client/v1/room.py
* Validate that the app service can actually control the given user
See https://github.com/matrix-org/synapse/pull/10276#issuecomment-876316455
Conflicts:
synapse/rest/client/v1/room.py
* Add some better comments on what we're trying to check for
* Continue debugging
* Share validation logic
* Add inserted historical messages to /backfill response
* Remove debug sql queries
* Some marker event implemntation trials
* Clean up PR
* Rename insertion_event_id to just event_id
* Add some better sql comments
* More accurate description
* Add changelog
* Make it clear what MSC the change is part of
* Add more detail on which insertion event came through
* Address review and improve sql queries
* Only use event_id as unique constraint
* Fix test case where insertion event is already in the normal DAG
* Remove debug changes
* Add support for MSC2716 marker events
* Process markers when we receive it over federation
* WIP: make hs2 backfill historical messages after marker event
* hs2 to better ask for insertion event extremity
But running into the `sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group`
error
* Add insertion_event_extremities table
* Switch to chunk events so we can auth via power_levels
Previously, we were using `content.chunk_id` to connect one
chunk to another. But these events can be from any `sender`
and we can't tell who should be able to send historical events.
We know we only want the application service to do it but these
events have the sender of a real historical message, not the
application service user ID as the sender. Other federated homeservers
also have no indicator which senders are an application service on
the originating homeserver.
So we want to auth all of the MSC2716 events via power_levels
and have them be sent by the application service with proper
PL levels in the room.
* Switch to chunk events for federation
* Add unstable room version to support new historical PL
* Messy: Fix undefined state_group for federated historical events
```
2021-07-13 02:27:57,810 - synapse.handlers.federation - 1248 - ERROR - GET-4 - Failed to backfill from hs1 because NOT NULL constraint failed: event_to_state_groups.state_group
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1216, in try_backfill
await self.backfill(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1035, in backfill
await self._auth_and_persist_event(dest, event, context, backfilled=True)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2222, in _auth_and_persist_event
await self._run_push_actions_and_persist_event(event, context, backfilled)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2244, in _run_push_actions_and_persist_event
await self.persist_events_and_notify(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 3290, in persist_events_and_notify
events, max_stream_token = await self.storage.persistence.persist_events(
File "/usr/local/lib/python3.8/site-packages/synapse/logging/opentracing.py", line 774, in _trace_inner
return await func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 320, in persist_events
ret_vals = await yieldable_gather_results(enqueue, partitioned.items())
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 237, in handle_queue_loop
ret = await self._per_item_callback(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 577, in _persist_event_batch
await self.persist_events_store._persist_events_and_state_updates(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 176, in _persist_events_and_state_updates
await self.db_pool.runInteraction(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 681, in runInteraction
result = await self.runWithConnection(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 770, in runWithConnection
return await make_deferred_yieldable(
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 238, in inContext
result = inContext.theWork() # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 254, in <lambda>
inContext.theWork = lambda: context.call( # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 83, in callWithContext
return func(*args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 293, in _runWithConnection
compat.reraise(excValue, excTraceback)
File "/usr/local/lib/python3.8/site-packages/twisted/python/deprecate.py", line 298, in deprecatedFunction
return function(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/twisted/python/compat.py", line 403, in reraise
raise exception.with_traceback(traceback)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 284, in _runWithConnection
result = func(conn, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 765, in inner_func
return func(db_conn, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 549, in new_transaction
r = func(cursor, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/logging/utils.py", line 69, in wrapped
return f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 385, in _persist_events_txn
self._store_event_state_mappings_txn(txn, events_and_contexts)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 2065, in _store_event_state_mappings_txn
self.db_pool.simple_insert_many_txn(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 923, in simple_insert_many_txn
txn.execute_batch(sql, vals)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 280, in execute_batch
self.executemany(sql, args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 300, in executemany
self._do_execute(self.txn.executemany, sql, *args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 330, in _do_execute
return func(sql, *args)
sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group
```
* Revert "Messy: Fix undefined state_group for federated historical events"
This reverts commit 187ab28611546321e02770944c86f30ee2bc742a.
* Fix federated events being rejected for no state_groups
Add fix from https://github.com/matrix-org/synapse/pull/10439
until it merges.
* Adapting to experimental room version
* Some log cleanup
* Add better comments around extremity fetching code and why
* Rename to be more accurate to what the function returns
* Add changelog
* Ignore rejected events
* Use simplified upsert
* Add Erik's explanation of extra event checks
See https://github.com/matrix-org/synapse/pull/10498#discussion_r680880332
* Clarify that the depth is not directly correlated to the backwards extremity that we return
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681725404
* lock only matters for sqlite
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681728061
* Move new SQL changes to its own delta file
* Clean up upsert docstring
* Bump database schema version (62)
2021-08-04 17:07:57 +00:00
|
|
|
|
"""
|
|
|
|
|
|
2022-09-23 19:01:29 +00:00
|
|
|
|
def get_backfill_points_in_room_txn(
|
2022-05-18 15:02:10 +00:00
|
|
|
|
txn: LoggingTransaction, room_id: str
|
|
|
|
|
) -> List[Tuple[str, int]]:
|
2022-09-23 19:01:29 +00:00
|
|
|
|
# Assemble a tuple lookup of event_id -> depth for the oldest events
|
Add support for MSC2716 marker events (#10498)
* Make historical messages available to federated servers
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
Follow-up to https://github.com/matrix-org/synapse/pull/9247
* Debug message not available on federation
* Add base starting insertion point when no chunk ID is provided
* Fix messages from multiple senders in historical chunk
Follow-up to https://github.com/matrix-org/synapse/pull/9247
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
---
Previously, Synapse would throw a 403,
`Cannot force another user to join.`,
because we were trying to use `?user_id` from a single virtual user
which did not match with messages from other users in the chunk.
* Remove debug lines
* Messing with selecting insertion event extremeties
* Move db schema change to new version
* Add more better comments
* Make a fake requester with just what we need
See https://github.com/matrix-org/synapse/pull/10276#discussion_r660999080
* Store insertion events in table
* Make base insertion event float off on its own
See https://github.com/matrix-org/synapse/pull/10250#issuecomment-875711889
Conflicts:
synapse/rest/client/v1/room.py
* Validate that the app service can actually control the given user
See https://github.com/matrix-org/synapse/pull/10276#issuecomment-876316455
Conflicts:
synapse/rest/client/v1/room.py
* Add some better comments on what we're trying to check for
* Continue debugging
* Share validation logic
* Add inserted historical messages to /backfill response
* Remove debug sql queries
* Some marker event implemntation trials
* Clean up PR
* Rename insertion_event_id to just event_id
* Add some better sql comments
* More accurate description
* Add changelog
* Make it clear what MSC the change is part of
* Add more detail on which insertion event came through
* Address review and improve sql queries
* Only use event_id as unique constraint
* Fix test case where insertion event is already in the normal DAG
* Remove debug changes
* Add support for MSC2716 marker events
* Process markers when we receive it over federation
* WIP: make hs2 backfill historical messages after marker event
* hs2 to better ask for insertion event extremity
But running into the `sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group`
error
* Add insertion_event_extremities table
* Switch to chunk events so we can auth via power_levels
Previously, we were using `content.chunk_id` to connect one
chunk to another. But these events can be from any `sender`
and we can't tell who should be able to send historical events.
We know we only want the application service to do it but these
events have the sender of a real historical message, not the
application service user ID as the sender. Other federated homeservers
also have no indicator which senders are an application service on
the originating homeserver.
So we want to auth all of the MSC2716 events via power_levels
and have them be sent by the application service with proper
PL levels in the room.
* Switch to chunk events for federation
* Add unstable room version to support new historical PL
* Messy: Fix undefined state_group for federated historical events
```
2021-07-13 02:27:57,810 - synapse.handlers.federation - 1248 - ERROR - GET-4 - Failed to backfill from hs1 because NOT NULL constraint failed: event_to_state_groups.state_group
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1216, in try_backfill
await self.backfill(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1035, in backfill
await self._auth_and_persist_event(dest, event, context, backfilled=True)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2222, in _auth_and_persist_event
await self._run_push_actions_and_persist_event(event, context, backfilled)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2244, in _run_push_actions_and_persist_event
await self.persist_events_and_notify(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 3290, in persist_events_and_notify
events, max_stream_token = await self.storage.persistence.persist_events(
File "/usr/local/lib/python3.8/site-packages/synapse/logging/opentracing.py", line 774, in _trace_inner
return await func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 320, in persist_events
ret_vals = await yieldable_gather_results(enqueue, partitioned.items())
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 237, in handle_queue_loop
ret = await self._per_item_callback(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 577, in _persist_event_batch
await self.persist_events_store._persist_events_and_state_updates(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 176, in _persist_events_and_state_updates
await self.db_pool.runInteraction(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 681, in runInteraction
result = await self.runWithConnection(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 770, in runWithConnection
return await make_deferred_yieldable(
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 238, in inContext
result = inContext.theWork() # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 254, in <lambda>
inContext.theWork = lambda: context.call( # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 83, in callWithContext
return func(*args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 293, in _runWithConnection
compat.reraise(excValue, excTraceback)
File "/usr/local/lib/python3.8/site-packages/twisted/python/deprecate.py", line 298, in deprecatedFunction
return function(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/twisted/python/compat.py", line 403, in reraise
raise exception.with_traceback(traceback)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 284, in _runWithConnection
result = func(conn, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 765, in inner_func
return func(db_conn, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 549, in new_transaction
r = func(cursor, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/logging/utils.py", line 69, in wrapped
return f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 385, in _persist_events_txn
self._store_event_state_mappings_txn(txn, events_and_contexts)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 2065, in _store_event_state_mappings_txn
self.db_pool.simple_insert_many_txn(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 923, in simple_insert_many_txn
txn.execute_batch(sql, vals)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 280, in execute_batch
self.executemany(sql, args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 300, in executemany
self._do_execute(self.txn.executemany, sql, *args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 330, in _do_execute
return func(sql, *args)
sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group
```
* Revert "Messy: Fix undefined state_group for federated historical events"
This reverts commit 187ab28611546321e02770944c86f30ee2bc742a.
* Fix federated events being rejected for no state_groups
Add fix from https://github.com/matrix-org/synapse/pull/10439
until it merges.
* Adapting to experimental room version
* Some log cleanup
* Add better comments around extremity fetching code and why
* Rename to be more accurate to what the function returns
* Add changelog
* Ignore rejected events
* Use simplified upsert
* Add Erik's explanation of extra event checks
See https://github.com/matrix-org/synapse/pull/10498#discussion_r680880332
* Clarify that the depth is not directly correlated to the backwards extremity that we return
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681725404
* lock only matters for sqlite
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681728061
* Move new SQL changes to its own delta file
* Clean up upsert docstring
* Bump database schema version (62)
2021-08-04 17:07:57 +00:00
|
|
|
|
# we know of in the room. Backwards extremeties are the oldest
|
|
|
|
|
# events we know of in the room but we only know of them because
|
2022-09-23 19:01:29 +00:00
|
|
|
|
# some other event referenced them by prev_event and aren't
|
|
|
|
|
# persisted in our database yet (meaning we don't know their depth
|
|
|
|
|
# specifically). So we need to look for the approximate depth from
|
Add support for MSC2716 marker events (#10498)
* Make historical messages available to federated servers
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
Follow-up to https://github.com/matrix-org/synapse/pull/9247
* Debug message not available on federation
* Add base starting insertion point when no chunk ID is provided
* Fix messages from multiple senders in historical chunk
Follow-up to https://github.com/matrix-org/synapse/pull/9247
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
---
Previously, Synapse would throw a 403,
`Cannot force another user to join.`,
because we were trying to use `?user_id` from a single virtual user
which did not match with messages from other users in the chunk.
* Remove debug lines
* Messing with selecting insertion event extremeties
* Move db schema change to new version
* Add more better comments
* Make a fake requester with just what we need
See https://github.com/matrix-org/synapse/pull/10276#discussion_r660999080
* Store insertion events in table
* Make base insertion event float off on its own
See https://github.com/matrix-org/synapse/pull/10250#issuecomment-875711889
Conflicts:
synapse/rest/client/v1/room.py
* Validate that the app service can actually control the given user
See https://github.com/matrix-org/synapse/pull/10276#issuecomment-876316455
Conflicts:
synapse/rest/client/v1/room.py
* Add some better comments on what we're trying to check for
* Continue debugging
* Share validation logic
* Add inserted historical messages to /backfill response
* Remove debug sql queries
* Some marker event implemntation trials
* Clean up PR
* Rename insertion_event_id to just event_id
* Add some better sql comments
* More accurate description
* Add changelog
* Make it clear what MSC the change is part of
* Add more detail on which insertion event came through
* Address review and improve sql queries
* Only use event_id as unique constraint
* Fix test case where insertion event is already in the normal DAG
* Remove debug changes
* Add support for MSC2716 marker events
* Process markers when we receive it over federation
* WIP: make hs2 backfill historical messages after marker event
* hs2 to better ask for insertion event extremity
But running into the `sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group`
error
* Add insertion_event_extremities table
* Switch to chunk events so we can auth via power_levels
Previously, we were using `content.chunk_id` to connect one
chunk to another. But these events can be from any `sender`
and we can't tell who should be able to send historical events.
We know we only want the application service to do it but these
events have the sender of a real historical message, not the
application service user ID as the sender. Other federated homeservers
also have no indicator which senders are an application service on
the originating homeserver.
So we want to auth all of the MSC2716 events via power_levels
and have them be sent by the application service with proper
PL levels in the room.
* Switch to chunk events for federation
* Add unstable room version to support new historical PL
* Messy: Fix undefined state_group for federated historical events
```
2021-07-13 02:27:57,810 - synapse.handlers.federation - 1248 - ERROR - GET-4 - Failed to backfill from hs1 because NOT NULL constraint failed: event_to_state_groups.state_group
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1216, in try_backfill
await self.backfill(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1035, in backfill
await self._auth_and_persist_event(dest, event, context, backfilled=True)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2222, in _auth_and_persist_event
await self._run_push_actions_and_persist_event(event, context, backfilled)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2244, in _run_push_actions_and_persist_event
await self.persist_events_and_notify(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 3290, in persist_events_and_notify
events, max_stream_token = await self.storage.persistence.persist_events(
File "/usr/local/lib/python3.8/site-packages/synapse/logging/opentracing.py", line 774, in _trace_inner
return await func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 320, in persist_events
ret_vals = await yieldable_gather_results(enqueue, partitioned.items())
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 237, in handle_queue_loop
ret = await self._per_item_callback(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 577, in _persist_event_batch
await self.persist_events_store._persist_events_and_state_updates(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 176, in _persist_events_and_state_updates
await self.db_pool.runInteraction(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 681, in runInteraction
result = await self.runWithConnection(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 770, in runWithConnection
return await make_deferred_yieldable(
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 238, in inContext
result = inContext.theWork() # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 254, in <lambda>
inContext.theWork = lambda: context.call( # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 83, in callWithContext
return func(*args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 293, in _runWithConnection
compat.reraise(excValue, excTraceback)
File "/usr/local/lib/python3.8/site-packages/twisted/python/deprecate.py", line 298, in deprecatedFunction
return function(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/twisted/python/compat.py", line 403, in reraise
raise exception.with_traceback(traceback)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 284, in _runWithConnection
result = func(conn, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 765, in inner_func
return func(db_conn, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 549, in new_transaction
r = func(cursor, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/logging/utils.py", line 69, in wrapped
return f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 385, in _persist_events_txn
self._store_event_state_mappings_txn(txn, events_and_contexts)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 2065, in _store_event_state_mappings_txn
self.db_pool.simple_insert_many_txn(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 923, in simple_insert_many_txn
txn.execute_batch(sql, vals)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 280, in execute_batch
self.executemany(sql, args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 300, in executemany
self._do_execute(self.txn.executemany, sql, *args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 330, in _do_execute
return func(sql, *args)
sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group
```
* Revert "Messy: Fix undefined state_group for federated historical events"
This reverts commit 187ab28611546321e02770944c86f30ee2bc742a.
* Fix federated events being rejected for no state_groups
Add fix from https://github.com/matrix-org/synapse/pull/10439
until it merges.
* Adapting to experimental room version
* Some log cleanup
* Add better comments around extremity fetching code and why
* Rename to be more accurate to what the function returns
* Add changelog
* Ignore rejected events
* Use simplified upsert
* Add Erik's explanation of extra event checks
See https://github.com/matrix-org/synapse/pull/10498#discussion_r680880332
* Clarify that the depth is not directly correlated to the backwards extremity that we return
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681725404
* lock only matters for sqlite
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681728061
* Move new SQL changes to its own delta file
* Clean up upsert docstring
* Bump database schema version (62)
2021-08-04 17:07:57 +00:00
|
|
|
|
# the events connected to the current backwards extremeties.
|
2022-09-30 10:54:53 +00:00
|
|
|
|
|
|
|
|
|
if isinstance(self.database_engine, PostgresEngine):
|
|
|
|
|
least_function = "LEAST"
|
|
|
|
|
elif isinstance(self.database_engine, Sqlite3Engine):
|
|
|
|
|
least_function = "MIN"
|
|
|
|
|
else:
|
|
|
|
|
raise RuntimeError("Unknown database engine")
|
|
|
|
|
|
|
|
|
|
sql = f"""
|
2022-09-23 19:01:29 +00:00
|
|
|
|
SELECT backward_extrem.event_id, event.depth FROM events AS event
|
Add support for MSC2716 marker events (#10498)
* Make historical messages available to federated servers
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
Follow-up to https://github.com/matrix-org/synapse/pull/9247
* Debug message not available on federation
* Add base starting insertion point when no chunk ID is provided
* Fix messages from multiple senders in historical chunk
Follow-up to https://github.com/matrix-org/synapse/pull/9247
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
---
Previously, Synapse would throw a 403,
`Cannot force another user to join.`,
because we were trying to use `?user_id` from a single virtual user
which did not match with messages from other users in the chunk.
* Remove debug lines
* Messing with selecting insertion event extremeties
* Move db schema change to new version
* Add more better comments
* Make a fake requester with just what we need
See https://github.com/matrix-org/synapse/pull/10276#discussion_r660999080
* Store insertion events in table
* Make base insertion event float off on its own
See https://github.com/matrix-org/synapse/pull/10250#issuecomment-875711889
Conflicts:
synapse/rest/client/v1/room.py
* Validate that the app service can actually control the given user
See https://github.com/matrix-org/synapse/pull/10276#issuecomment-876316455
Conflicts:
synapse/rest/client/v1/room.py
* Add some better comments on what we're trying to check for
* Continue debugging
* Share validation logic
* Add inserted historical messages to /backfill response
* Remove debug sql queries
* Some marker event implemntation trials
* Clean up PR
* Rename insertion_event_id to just event_id
* Add some better sql comments
* More accurate description
* Add changelog
* Make it clear what MSC the change is part of
* Add more detail on which insertion event came through
* Address review and improve sql queries
* Only use event_id as unique constraint
* Fix test case where insertion event is already in the normal DAG
* Remove debug changes
* Add support for MSC2716 marker events
* Process markers when we receive it over federation
* WIP: make hs2 backfill historical messages after marker event
* hs2 to better ask for insertion event extremity
But running into the `sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group`
error
* Add insertion_event_extremities table
* Switch to chunk events so we can auth via power_levels
Previously, we were using `content.chunk_id` to connect one
chunk to another. But these events can be from any `sender`
and we can't tell who should be able to send historical events.
We know we only want the application service to do it but these
events have the sender of a real historical message, not the
application service user ID as the sender. Other federated homeservers
also have no indicator which senders are an application service on
the originating homeserver.
So we want to auth all of the MSC2716 events via power_levels
and have them be sent by the application service with proper
PL levels in the room.
* Switch to chunk events for federation
* Add unstable room version to support new historical PL
* Messy: Fix undefined state_group for federated historical events
```
2021-07-13 02:27:57,810 - synapse.handlers.federation - 1248 - ERROR - GET-4 - Failed to backfill from hs1 because NOT NULL constraint failed: event_to_state_groups.state_group
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1216, in try_backfill
await self.backfill(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1035, in backfill
await self._auth_and_persist_event(dest, event, context, backfilled=True)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2222, in _auth_and_persist_event
await self._run_push_actions_and_persist_event(event, context, backfilled)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2244, in _run_push_actions_and_persist_event
await self.persist_events_and_notify(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 3290, in persist_events_and_notify
events, max_stream_token = await self.storage.persistence.persist_events(
File "/usr/local/lib/python3.8/site-packages/synapse/logging/opentracing.py", line 774, in _trace_inner
return await func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 320, in persist_events
ret_vals = await yieldable_gather_results(enqueue, partitioned.items())
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 237, in handle_queue_loop
ret = await self._per_item_callback(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 577, in _persist_event_batch
await self.persist_events_store._persist_events_and_state_updates(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 176, in _persist_events_and_state_updates
await self.db_pool.runInteraction(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 681, in runInteraction
result = await self.runWithConnection(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 770, in runWithConnection
return await make_deferred_yieldable(
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 238, in inContext
result = inContext.theWork() # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 254, in <lambda>
inContext.theWork = lambda: context.call( # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 83, in callWithContext
return func(*args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 293, in _runWithConnection
compat.reraise(excValue, excTraceback)
File "/usr/local/lib/python3.8/site-packages/twisted/python/deprecate.py", line 298, in deprecatedFunction
return function(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/twisted/python/compat.py", line 403, in reraise
raise exception.with_traceback(traceback)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 284, in _runWithConnection
result = func(conn, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 765, in inner_func
return func(db_conn, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 549, in new_transaction
r = func(cursor, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/logging/utils.py", line 69, in wrapped
return f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 385, in _persist_events_txn
self._store_event_state_mappings_txn(txn, events_and_contexts)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 2065, in _store_event_state_mappings_txn
self.db_pool.simple_insert_many_txn(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 923, in simple_insert_many_txn
txn.execute_batch(sql, vals)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 280, in execute_batch
self.executemany(sql, args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 300, in executemany
self._do_execute(self.txn.executemany, sql, *args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 330, in _do_execute
return func(sql, *args)
sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group
```
* Revert "Messy: Fix undefined state_group for federated historical events"
This reverts commit 187ab28611546321e02770944c86f30ee2bc742a.
* Fix federated events being rejected for no state_groups
Add fix from https://github.com/matrix-org/synapse/pull/10439
until it merges.
* Adapting to experimental room version
* Some log cleanup
* Add better comments around extremity fetching code and why
* Rename to be more accurate to what the function returns
* Add changelog
* Ignore rejected events
* Use simplified upsert
* Add Erik's explanation of extra event checks
See https://github.com/matrix-org/synapse/pull/10498#discussion_r680880332
* Clarify that the depth is not directly correlated to the backwards extremity that we return
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681725404
* lock only matters for sqlite
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681728061
* Move new SQL changes to its own delta file
* Clean up upsert docstring
* Bump database schema version (62)
2021-08-04 17:07:57 +00:00
|
|
|
|
/**
|
|
|
|
|
* Get the edge connections from the event_edges table
|
|
|
|
|
* so we can see whether this event's prev_events points
|
|
|
|
|
* to a backward extremity in the next join.
|
|
|
|
|
*/
|
2022-09-23 19:01:29 +00:00
|
|
|
|
INNER JOIN event_edges AS edge
|
|
|
|
|
ON edge.event_id = event.event_id
|
Add support for MSC2716 marker events (#10498)
* Make historical messages available to federated servers
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
Follow-up to https://github.com/matrix-org/synapse/pull/9247
* Debug message not available on federation
* Add base starting insertion point when no chunk ID is provided
* Fix messages from multiple senders in historical chunk
Follow-up to https://github.com/matrix-org/synapse/pull/9247
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
---
Previously, Synapse would throw a 403,
`Cannot force another user to join.`,
because we were trying to use `?user_id` from a single virtual user
which did not match with messages from other users in the chunk.
* Remove debug lines
* Messing with selecting insertion event extremeties
* Move db schema change to new version
* Add more better comments
* Make a fake requester with just what we need
See https://github.com/matrix-org/synapse/pull/10276#discussion_r660999080
* Store insertion events in table
* Make base insertion event float off on its own
See https://github.com/matrix-org/synapse/pull/10250#issuecomment-875711889
Conflicts:
synapse/rest/client/v1/room.py
* Validate that the app service can actually control the given user
See https://github.com/matrix-org/synapse/pull/10276#issuecomment-876316455
Conflicts:
synapse/rest/client/v1/room.py
* Add some better comments on what we're trying to check for
* Continue debugging
* Share validation logic
* Add inserted historical messages to /backfill response
* Remove debug sql queries
* Some marker event implemntation trials
* Clean up PR
* Rename insertion_event_id to just event_id
* Add some better sql comments
* More accurate description
* Add changelog
* Make it clear what MSC the change is part of
* Add more detail on which insertion event came through
* Address review and improve sql queries
* Only use event_id as unique constraint
* Fix test case where insertion event is already in the normal DAG
* Remove debug changes
* Add support for MSC2716 marker events
* Process markers when we receive it over federation
* WIP: make hs2 backfill historical messages after marker event
* hs2 to better ask for insertion event extremity
But running into the `sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group`
error
* Add insertion_event_extremities table
* Switch to chunk events so we can auth via power_levels
Previously, we were using `content.chunk_id` to connect one
chunk to another. But these events can be from any `sender`
and we can't tell who should be able to send historical events.
We know we only want the application service to do it but these
events have the sender of a real historical message, not the
application service user ID as the sender. Other federated homeservers
also have no indicator which senders are an application service on
the originating homeserver.
So we want to auth all of the MSC2716 events via power_levels
and have them be sent by the application service with proper
PL levels in the room.
* Switch to chunk events for federation
* Add unstable room version to support new historical PL
* Messy: Fix undefined state_group for federated historical events
```
2021-07-13 02:27:57,810 - synapse.handlers.federation - 1248 - ERROR - GET-4 - Failed to backfill from hs1 because NOT NULL constraint failed: event_to_state_groups.state_group
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1216, in try_backfill
await self.backfill(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1035, in backfill
await self._auth_and_persist_event(dest, event, context, backfilled=True)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2222, in _auth_and_persist_event
await self._run_push_actions_and_persist_event(event, context, backfilled)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2244, in _run_push_actions_and_persist_event
await self.persist_events_and_notify(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 3290, in persist_events_and_notify
events, max_stream_token = await self.storage.persistence.persist_events(
File "/usr/local/lib/python3.8/site-packages/synapse/logging/opentracing.py", line 774, in _trace_inner
return await func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 320, in persist_events
ret_vals = await yieldable_gather_results(enqueue, partitioned.items())
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 237, in handle_queue_loop
ret = await self._per_item_callback(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 577, in _persist_event_batch
await self.persist_events_store._persist_events_and_state_updates(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 176, in _persist_events_and_state_updates
await self.db_pool.runInteraction(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 681, in runInteraction
result = await self.runWithConnection(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 770, in runWithConnection
return await make_deferred_yieldable(
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 238, in inContext
result = inContext.theWork() # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 254, in <lambda>
inContext.theWork = lambda: context.call( # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 83, in callWithContext
return func(*args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 293, in _runWithConnection
compat.reraise(excValue, excTraceback)
File "/usr/local/lib/python3.8/site-packages/twisted/python/deprecate.py", line 298, in deprecatedFunction
return function(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/twisted/python/compat.py", line 403, in reraise
raise exception.with_traceback(traceback)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 284, in _runWithConnection
result = func(conn, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 765, in inner_func
return func(db_conn, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 549, in new_transaction
r = func(cursor, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/logging/utils.py", line 69, in wrapped
return f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 385, in _persist_events_txn
self._store_event_state_mappings_txn(txn, events_and_contexts)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 2065, in _store_event_state_mappings_txn
self.db_pool.simple_insert_many_txn(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 923, in simple_insert_many_txn
txn.execute_batch(sql, vals)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 280, in execute_batch
self.executemany(sql, args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 300, in executemany
self._do_execute(self.txn.executemany, sql, *args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 330, in _do_execute
return func(sql, *args)
sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group
```
* Revert "Messy: Fix undefined state_group for federated historical events"
This reverts commit 187ab28611546321e02770944c86f30ee2bc742a.
* Fix federated events being rejected for no state_groups
Add fix from https://github.com/matrix-org/synapse/pull/10439
until it merges.
* Adapting to experimental room version
* Some log cleanup
* Add better comments around extremity fetching code and why
* Rename to be more accurate to what the function returns
* Add changelog
* Ignore rejected events
* Use simplified upsert
* Add Erik's explanation of extra event checks
See https://github.com/matrix-org/synapse/pull/10498#discussion_r680880332
* Clarify that the depth is not directly correlated to the backwards extremity that we return
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681725404
* lock only matters for sqlite
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681728061
* Move new SQL changes to its own delta file
* Clean up upsert docstring
* Bump database schema version (62)
2021-08-04 17:07:57 +00:00
|
|
|
|
/**
|
|
|
|
|
* We find the "oldest" events in the room by looking for
|
|
|
|
|
* events connected to backwards extremeties (oldest events
|
|
|
|
|
* in the room that we know of so far).
|
|
|
|
|
*/
|
2022-09-23 19:01:29 +00:00
|
|
|
|
INNER JOIN event_backward_extremities AS backward_extrem
|
|
|
|
|
ON edge.prev_event_id = backward_extrem.event_id
|
|
|
|
|
/**
|
|
|
|
|
* We use this info to make sure we don't retry to use a backfill point
|
|
|
|
|
* if we've already attempted to backfill from it recently.
|
|
|
|
|
*/
|
|
|
|
|
LEFT JOIN event_failed_pull_attempts AS failed_backfill_attempt_info
|
|
|
|
|
ON
|
|
|
|
|
failed_backfill_attempt_info.room_id = backward_extrem.room_id
|
|
|
|
|
AND failed_backfill_attempt_info.event_id = backward_extrem.event_id
|
|
|
|
|
WHERE
|
|
|
|
|
backward_extrem.room_id = ?
|
|
|
|
|
/* We only care about non-state edges because we used to use
|
|
|
|
|
* `event_edges` for two different sorts of "edges" (the current
|
|
|
|
|
* event DAG, but also a link to the previous state, for state
|
|
|
|
|
* events). These legacy state event edges can be distinguished by
|
|
|
|
|
* `is_state` and are removed from the codebase and schema but
|
|
|
|
|
* because the schema change is in a background update, it's not
|
|
|
|
|
* necessarily safe to assume that it will have been completed.
|
|
|
|
|
*/
|
|
|
|
|
AND edge.is_state is ? /* False */
|
2022-09-28 20:26:16 +00:00
|
|
|
|
/**
|
|
|
|
|
* We only want backwards extremities that are older than or at
|
|
|
|
|
* the same position of the given `current_depth` (where older
|
|
|
|
|
* means less than the given depth) because we're looking backwards
|
|
|
|
|
* from the `current_depth` when backfilling.
|
|
|
|
|
*
|
|
|
|
|
* current_depth (ignore events that come after this, ignore 2-4)
|
|
|
|
|
* |
|
|
|
|
|
* ▼
|
|
|
|
|
* <oldest-in-time> [0]<--[1]<--[2]<--[3]<--[4] <newest-in-time>
|
|
|
|
|
*/
|
|
|
|
|
AND event.depth <= ? /* current_depth */
|
2022-09-23 19:01:29 +00:00
|
|
|
|
/**
|
|
|
|
|
* Exponential back-off (up to the upper bound) so we don't retry the
|
|
|
|
|
* same backfill point over and over. ex. 2hr, 4hr, 8hr, 16hr, etc.
|
|
|
|
|
*
|
|
|
|
|
* We use `1 << n` as a power of 2 equivalent for compatibility
|
|
|
|
|
* with older SQLites. The left shift equivalent only works with
|
|
|
|
|
* powers of 2 because left shift is a binary operation (base-2).
|
|
|
|
|
* Otherwise, we would use `power(2, n)` or the power operator, `2^n`.
|
|
|
|
|
*/
|
|
|
|
|
AND (
|
|
|
|
|
failed_backfill_attempt_info.event_id IS NULL
|
2022-09-30 10:54:53 +00:00
|
|
|
|
OR ? /* current_time */ >= failed_backfill_attempt_info.last_attempt_ts + (
|
|
|
|
|
(1 << {least_function}(failed_backfill_attempt_info.num_attempts, ? /* max doubling steps */))
|
|
|
|
|
* ? /* step */
|
|
|
|
|
)
|
2022-09-23 19:01:29 +00:00
|
|
|
|
)
|
|
|
|
|
/**
|
2022-09-28 20:26:16 +00:00
|
|
|
|
* Sort from highest (closest to the `current_depth`) to the lowest depth
|
|
|
|
|
* because the closest are most relevant to backfill from first.
|
|
|
|
|
* Then tie-break on alphabetical order of the event_ids so we get a
|
|
|
|
|
* consistent ordering which is nice when asserting things in tests.
|
2022-09-23 19:01:29 +00:00
|
|
|
|
*/
|
|
|
|
|
ORDER BY event.depth DESC, backward_extrem.event_id DESC
|
2022-09-28 20:26:16 +00:00
|
|
|
|
LIMIT ?
|
Add support for MSC2716 marker events (#10498)
* Make historical messages available to federated servers
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
Follow-up to https://github.com/matrix-org/synapse/pull/9247
* Debug message not available on federation
* Add base starting insertion point when no chunk ID is provided
* Fix messages from multiple senders in historical chunk
Follow-up to https://github.com/matrix-org/synapse/pull/9247
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
---
Previously, Synapse would throw a 403,
`Cannot force another user to join.`,
because we were trying to use `?user_id` from a single virtual user
which did not match with messages from other users in the chunk.
* Remove debug lines
* Messing with selecting insertion event extremeties
* Move db schema change to new version
* Add more better comments
* Make a fake requester with just what we need
See https://github.com/matrix-org/synapse/pull/10276#discussion_r660999080
* Store insertion events in table
* Make base insertion event float off on its own
See https://github.com/matrix-org/synapse/pull/10250#issuecomment-875711889
Conflicts:
synapse/rest/client/v1/room.py
* Validate that the app service can actually control the given user
See https://github.com/matrix-org/synapse/pull/10276#issuecomment-876316455
Conflicts:
synapse/rest/client/v1/room.py
* Add some better comments on what we're trying to check for
* Continue debugging
* Share validation logic
* Add inserted historical messages to /backfill response
* Remove debug sql queries
* Some marker event implemntation trials
* Clean up PR
* Rename insertion_event_id to just event_id
* Add some better sql comments
* More accurate description
* Add changelog
* Make it clear what MSC the change is part of
* Add more detail on which insertion event came through
* Address review and improve sql queries
* Only use event_id as unique constraint
* Fix test case where insertion event is already in the normal DAG
* Remove debug changes
* Add support for MSC2716 marker events
* Process markers when we receive it over federation
* WIP: make hs2 backfill historical messages after marker event
* hs2 to better ask for insertion event extremity
But running into the `sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group`
error
* Add insertion_event_extremities table
* Switch to chunk events so we can auth via power_levels
Previously, we were using `content.chunk_id` to connect one
chunk to another. But these events can be from any `sender`
and we can't tell who should be able to send historical events.
We know we only want the application service to do it but these
events have the sender of a real historical message, not the
application service user ID as the sender. Other federated homeservers
also have no indicator which senders are an application service on
the originating homeserver.
So we want to auth all of the MSC2716 events via power_levels
and have them be sent by the application service with proper
PL levels in the room.
* Switch to chunk events for federation
* Add unstable room version to support new historical PL
* Messy: Fix undefined state_group for federated historical events
```
2021-07-13 02:27:57,810 - synapse.handlers.federation - 1248 - ERROR - GET-4 - Failed to backfill from hs1 because NOT NULL constraint failed: event_to_state_groups.state_group
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1216, in try_backfill
await self.backfill(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1035, in backfill
await self._auth_and_persist_event(dest, event, context, backfilled=True)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2222, in _auth_and_persist_event
await self._run_push_actions_and_persist_event(event, context, backfilled)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2244, in _run_push_actions_and_persist_event
await self.persist_events_and_notify(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 3290, in persist_events_and_notify
events, max_stream_token = await self.storage.persistence.persist_events(
File "/usr/local/lib/python3.8/site-packages/synapse/logging/opentracing.py", line 774, in _trace_inner
return await func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 320, in persist_events
ret_vals = await yieldable_gather_results(enqueue, partitioned.items())
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 237, in handle_queue_loop
ret = await self._per_item_callback(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 577, in _persist_event_batch
await self.persist_events_store._persist_events_and_state_updates(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 176, in _persist_events_and_state_updates
await self.db_pool.runInteraction(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 681, in runInteraction
result = await self.runWithConnection(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 770, in runWithConnection
return await make_deferred_yieldable(
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 238, in inContext
result = inContext.theWork() # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 254, in <lambda>
inContext.theWork = lambda: context.call( # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 83, in callWithContext
return func(*args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 293, in _runWithConnection
compat.reraise(excValue, excTraceback)
File "/usr/local/lib/python3.8/site-packages/twisted/python/deprecate.py", line 298, in deprecatedFunction
return function(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/twisted/python/compat.py", line 403, in reraise
raise exception.with_traceback(traceback)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 284, in _runWithConnection
result = func(conn, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 765, in inner_func
return func(db_conn, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 549, in new_transaction
r = func(cursor, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/logging/utils.py", line 69, in wrapped
return f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 385, in _persist_events_txn
self._store_event_state_mappings_txn(txn, events_and_contexts)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 2065, in _store_event_state_mappings_txn
self.db_pool.simple_insert_many_txn(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 923, in simple_insert_many_txn
txn.execute_batch(sql, vals)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 280, in execute_batch
self.executemany(sql, args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 300, in executemany
self._do_execute(self.txn.executemany, sql, *args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 330, in _do_execute
return func(sql, *args)
sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group
```
* Revert "Messy: Fix undefined state_group for federated historical events"
This reverts commit 187ab28611546321e02770944c86f30ee2bc742a.
* Fix federated events being rejected for no state_groups
Add fix from https://github.com/matrix-org/synapse/pull/10439
until it merges.
* Adapting to experimental room version
* Some log cleanup
* Add better comments around extremity fetching code and why
* Rename to be more accurate to what the function returns
* Add changelog
* Ignore rejected events
* Use simplified upsert
* Add Erik's explanation of extra event checks
See https://github.com/matrix-org/synapse/pull/10498#discussion_r680880332
* Clarify that the depth is not directly correlated to the backwards extremity that we return
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681725404
* lock only matters for sqlite
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681728061
* Move new SQL changes to its own delta file
* Clean up upsert docstring
* Bump database schema version (62)
2021-08-04 17:07:57 +00:00
|
|
|
|
"""
|
|
|
|
|
|
2022-09-23 19:01:29 +00:00
|
|
|
|
txn.execute(
|
2022-09-30 10:54:53 +00:00
|
|
|
|
sql,
|
2022-09-23 19:01:29 +00:00
|
|
|
|
(
|
|
|
|
|
room_id,
|
|
|
|
|
False,
|
2022-09-28 20:26:16 +00:00
|
|
|
|
current_depth,
|
2022-09-23 19:01:29 +00:00
|
|
|
|
self._clock.time_msec(),
|
2022-09-30 10:54:53 +00:00
|
|
|
|
BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS,
|
|
|
|
|
BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS,
|
2022-09-28 20:26:16 +00:00
|
|
|
|
limit,
|
2022-09-23 19:01:29 +00:00
|
|
|
|
),
|
|
|
|
|
)
|
Add support for MSC2716 marker events (#10498)
* Make historical messages available to federated servers
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
Follow-up to https://github.com/matrix-org/synapse/pull/9247
* Debug message not available on federation
* Add base starting insertion point when no chunk ID is provided
* Fix messages from multiple senders in historical chunk
Follow-up to https://github.com/matrix-org/synapse/pull/9247
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
---
Previously, Synapse would throw a 403,
`Cannot force another user to join.`,
because we were trying to use `?user_id` from a single virtual user
which did not match with messages from other users in the chunk.
* Remove debug lines
* Messing with selecting insertion event extremeties
* Move db schema change to new version
* Add more better comments
* Make a fake requester with just what we need
See https://github.com/matrix-org/synapse/pull/10276#discussion_r660999080
* Store insertion events in table
* Make base insertion event float off on its own
See https://github.com/matrix-org/synapse/pull/10250#issuecomment-875711889
Conflicts:
synapse/rest/client/v1/room.py
* Validate that the app service can actually control the given user
See https://github.com/matrix-org/synapse/pull/10276#issuecomment-876316455
Conflicts:
synapse/rest/client/v1/room.py
* Add some better comments on what we're trying to check for
* Continue debugging
* Share validation logic
* Add inserted historical messages to /backfill response
* Remove debug sql queries
* Some marker event implemntation trials
* Clean up PR
* Rename insertion_event_id to just event_id
* Add some better sql comments
* More accurate description
* Add changelog
* Make it clear what MSC the change is part of
* Add more detail on which insertion event came through
* Address review and improve sql queries
* Only use event_id as unique constraint
* Fix test case where insertion event is already in the normal DAG
* Remove debug changes
* Add support for MSC2716 marker events
* Process markers when we receive it over federation
* WIP: make hs2 backfill historical messages after marker event
* hs2 to better ask for insertion event extremity
But running into the `sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group`
error
* Add insertion_event_extremities table
* Switch to chunk events so we can auth via power_levels
Previously, we were using `content.chunk_id` to connect one
chunk to another. But these events can be from any `sender`
and we can't tell who should be able to send historical events.
We know we only want the application service to do it but these
events have the sender of a real historical message, not the
application service user ID as the sender. Other federated homeservers
also have no indicator which senders are an application service on
the originating homeserver.
So we want to auth all of the MSC2716 events via power_levels
and have them be sent by the application service with proper
PL levels in the room.
* Switch to chunk events for federation
* Add unstable room version to support new historical PL
* Messy: Fix undefined state_group for federated historical events
```
2021-07-13 02:27:57,810 - synapse.handlers.federation - 1248 - ERROR - GET-4 - Failed to backfill from hs1 because NOT NULL constraint failed: event_to_state_groups.state_group
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1216, in try_backfill
await self.backfill(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1035, in backfill
await self._auth_and_persist_event(dest, event, context, backfilled=True)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2222, in _auth_and_persist_event
await self._run_push_actions_and_persist_event(event, context, backfilled)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2244, in _run_push_actions_and_persist_event
await self.persist_events_and_notify(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 3290, in persist_events_and_notify
events, max_stream_token = await self.storage.persistence.persist_events(
File "/usr/local/lib/python3.8/site-packages/synapse/logging/opentracing.py", line 774, in _trace_inner
return await func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 320, in persist_events
ret_vals = await yieldable_gather_results(enqueue, partitioned.items())
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 237, in handle_queue_loop
ret = await self._per_item_callback(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 577, in _persist_event_batch
await self.persist_events_store._persist_events_and_state_updates(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 176, in _persist_events_and_state_updates
await self.db_pool.runInteraction(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 681, in runInteraction
result = await self.runWithConnection(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 770, in runWithConnection
return await make_deferred_yieldable(
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 238, in inContext
result = inContext.theWork() # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 254, in <lambda>
inContext.theWork = lambda: context.call( # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 83, in callWithContext
return func(*args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 293, in _runWithConnection
compat.reraise(excValue, excTraceback)
File "/usr/local/lib/python3.8/site-packages/twisted/python/deprecate.py", line 298, in deprecatedFunction
return function(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/twisted/python/compat.py", line 403, in reraise
raise exception.with_traceback(traceback)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 284, in _runWithConnection
result = func(conn, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 765, in inner_func
return func(db_conn, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 549, in new_transaction
r = func(cursor, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/logging/utils.py", line 69, in wrapped
return f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 385, in _persist_events_txn
self._store_event_state_mappings_txn(txn, events_and_contexts)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 2065, in _store_event_state_mappings_txn
self.db_pool.simple_insert_many_txn(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 923, in simple_insert_many_txn
txn.execute_batch(sql, vals)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 280, in execute_batch
self.executemany(sql, args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 300, in executemany
self._do_execute(self.txn.executemany, sql, *args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 330, in _do_execute
return func(sql, *args)
sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group
```
* Revert "Messy: Fix undefined state_group for federated historical events"
This reverts commit 187ab28611546321e02770944c86f30ee2bc742a.
* Fix federated events being rejected for no state_groups
Add fix from https://github.com/matrix-org/synapse/pull/10439
until it merges.
* Adapting to experimental room version
* Some log cleanup
* Add better comments around extremity fetching code and why
* Rename to be more accurate to what the function returns
* Add changelog
* Ignore rejected events
* Use simplified upsert
* Add Erik's explanation of extra event checks
See https://github.com/matrix-org/synapse/pull/10498#discussion_r680880332
* Clarify that the depth is not directly correlated to the backwards extremity that we return
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681725404
* lock only matters for sqlite
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681728061
* Move new SQL changes to its own delta file
* Clean up upsert docstring
* Bump database schema version (62)
2021-08-04 17:07:57 +00:00
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
return cast(List[Tuple[str, int]], txn.fetchall())
|
Add support for MSC2716 marker events (#10498)
* Make historical messages available to federated servers
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
Follow-up to https://github.com/matrix-org/synapse/pull/9247
* Debug message not available on federation
* Add base starting insertion point when no chunk ID is provided
* Fix messages from multiple senders in historical chunk
Follow-up to https://github.com/matrix-org/synapse/pull/9247
Part of MSC2716: https://github.com/matrix-org/matrix-doc/pull/2716
---
Previously, Synapse would throw a 403,
`Cannot force another user to join.`,
because we were trying to use `?user_id` from a single virtual user
which did not match with messages from other users in the chunk.
* Remove debug lines
* Messing with selecting insertion event extremeties
* Move db schema change to new version
* Add more better comments
* Make a fake requester with just what we need
See https://github.com/matrix-org/synapse/pull/10276#discussion_r660999080
* Store insertion events in table
* Make base insertion event float off on its own
See https://github.com/matrix-org/synapse/pull/10250#issuecomment-875711889
Conflicts:
synapse/rest/client/v1/room.py
* Validate that the app service can actually control the given user
See https://github.com/matrix-org/synapse/pull/10276#issuecomment-876316455
Conflicts:
synapse/rest/client/v1/room.py
* Add some better comments on what we're trying to check for
* Continue debugging
* Share validation logic
* Add inserted historical messages to /backfill response
* Remove debug sql queries
* Some marker event implemntation trials
* Clean up PR
* Rename insertion_event_id to just event_id
* Add some better sql comments
* More accurate description
* Add changelog
* Make it clear what MSC the change is part of
* Add more detail on which insertion event came through
* Address review and improve sql queries
* Only use event_id as unique constraint
* Fix test case where insertion event is already in the normal DAG
* Remove debug changes
* Add support for MSC2716 marker events
* Process markers when we receive it over federation
* WIP: make hs2 backfill historical messages after marker event
* hs2 to better ask for insertion event extremity
But running into the `sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group`
error
* Add insertion_event_extremities table
* Switch to chunk events so we can auth via power_levels
Previously, we were using `content.chunk_id` to connect one
chunk to another. But these events can be from any `sender`
and we can't tell who should be able to send historical events.
We know we only want the application service to do it but these
events have the sender of a real historical message, not the
application service user ID as the sender. Other federated homeservers
also have no indicator which senders are an application service on
the originating homeserver.
So we want to auth all of the MSC2716 events via power_levels
and have them be sent by the application service with proper
PL levels in the room.
* Switch to chunk events for federation
* Add unstable room version to support new historical PL
* Messy: Fix undefined state_group for federated historical events
```
2021-07-13 02:27:57,810 - synapse.handlers.federation - 1248 - ERROR - GET-4 - Failed to backfill from hs1 because NOT NULL constraint failed: event_to_state_groups.state_group
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1216, in try_backfill
await self.backfill(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 1035, in backfill
await self._auth_and_persist_event(dest, event, context, backfilled=True)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2222, in _auth_and_persist_event
await self._run_push_actions_and_persist_event(event, context, backfilled)
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 2244, in _run_push_actions_and_persist_event
await self.persist_events_and_notify(
File "/usr/local/lib/python3.8/site-packages/synapse/handlers/federation.py", line 3290, in persist_events_and_notify
events, max_stream_token = await self.storage.persistence.persist_events(
File "/usr/local/lib/python3.8/site-packages/synapse/logging/opentracing.py", line 774, in _trace_inner
return await func(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 320, in persist_events
ret_vals = await yieldable_gather_results(enqueue, partitioned.items())
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 237, in handle_queue_loop
ret = await self._per_item_callback(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/persist_events.py", line 577, in _persist_event_batch
await self.persist_events_store._persist_events_and_state_updates(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 176, in _persist_events_and_state_updates
await self.db_pool.runInteraction(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 681, in runInteraction
result = await self.runWithConnection(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 770, in runWithConnection
return await make_deferred_yieldable(
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 238, in inContext
result = inContext.theWork() # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/threadpool.py", line 254, in <lambda>
inContext.theWork = lambda: context.call( # type: ignore[attr-defined]
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/python/context.py", line 83, in callWithContext
return func(*args, **kw)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 293, in _runWithConnection
compat.reraise(excValue, excTraceback)
File "/usr/local/lib/python3.8/site-packages/twisted/python/deprecate.py", line 298, in deprecatedFunction
return function(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/twisted/python/compat.py", line 403, in reraise
raise exception.with_traceback(traceback)
File "/usr/local/lib/python3.8/site-packages/twisted/enterprise/adbapi.py", line 284, in _runWithConnection
result = func(conn, *args, **kw)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 765, in inner_func
return func(db_conn, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 549, in new_transaction
r = func(cursor, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/logging/utils.py", line 69, in wrapped
return f(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 385, in _persist_events_txn
self._store_event_state_mappings_txn(txn, events_and_contexts)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/databases/main/events.py", line 2065, in _store_event_state_mappings_txn
self.db_pool.simple_insert_many_txn(
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 923, in simple_insert_many_txn
txn.execute_batch(sql, vals)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 280, in execute_batch
self.executemany(sql, args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 300, in executemany
self._do_execute(self.txn.executemany, sql, *args)
File "/usr/local/lib/python3.8/site-packages/synapse/storage/database.py", line 330, in _do_execute
return func(sql, *args)
sqlite3.IntegrityError: NOT NULL constraint failed: event_to_state_groups.state_group
```
* Revert "Messy: Fix undefined state_group for federated historical events"
This reverts commit 187ab28611546321e02770944c86f30ee2bc742a.
* Fix federated events being rejected for no state_groups
Add fix from https://github.com/matrix-org/synapse/pull/10439
until it merges.
* Adapting to experimental room version
* Some log cleanup
* Add better comments around extremity fetching code and why
* Rename to be more accurate to what the function returns
* Add changelog
* Ignore rejected events
* Use simplified upsert
* Add Erik's explanation of extra event checks
See https://github.com/matrix-org/synapse/pull/10498#discussion_r680880332
* Clarify that the depth is not directly correlated to the backwards extremity that we return
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681725404
* lock only matters for sqlite
See https://github.com/matrix-org/synapse/pull/10498#discussion_r681728061
* Move new SQL changes to its own delta file
* Clean up upsert docstring
* Bump database schema version (62)
2021-08-04 17:07:57 +00:00
|
|
|
|
|
2020-08-28 11:54:27 +00:00
|
|
|
|
return await self.db_pool.runInteraction(
|
2022-09-23 19:01:29 +00:00
|
|
|
|
"get_backfill_points_in_room",
|
|
|
|
|
get_backfill_points_in_room_txn,
|
2015-05-11 17:01:31 +00:00
|
|
|
|
room_id,
|
|
|
|
|
)
|
|
|
|
|
|
2023-02-10 23:29:00 +00:00
|
|
|
|
async def get_max_depth_of(
|
|
|
|
|
self, event_ids: Collection[str]
|
|
|
|
|
) -> Tuple[Optional[str], int]:
|
2021-06-22 09:02:53 +00:00
|
|
|
|
"""Returns the event ID and depth for the event that has the max depth from a set of event IDs
|
2019-01-25 17:19:31 +00:00
|
|
|
|
|
|
|
|
|
Args:
|
2020-08-11 21:21:13 +00:00
|
|
|
|
event_ids: The event IDs to calculate the max depth of.
|
2019-01-25 17:19:31 +00:00
|
|
|
|
"""
|
2020-08-11 21:21:13 +00:00
|
|
|
|
rows = await self.db_pool.simple_select_many_batch(
|
2019-01-25 17:19:31 +00:00
|
|
|
|
table="events",
|
|
|
|
|
column="event_id",
|
|
|
|
|
iterable=event_ids,
|
2021-06-22 09:02:53 +00:00
|
|
|
|
retcols=(
|
|
|
|
|
"event_id",
|
|
|
|
|
"depth",
|
|
|
|
|
),
|
2019-01-25 17:19:31 +00:00
|
|
|
|
desc="get_max_depth_of",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
if not rows:
|
2021-06-22 09:02:53 +00:00
|
|
|
|
return None, 0
|
2019-01-25 17:19:31 +00:00
|
|
|
|
else:
|
2021-06-22 09:02:53 +00:00
|
|
|
|
max_depth_event_id = ""
|
|
|
|
|
current_max_depth = 0
|
|
|
|
|
for row in rows:
|
|
|
|
|
if row["depth"] > current_max_depth:
|
|
|
|
|
max_depth_event_id = row["event_id"]
|
|
|
|
|
current_max_depth = row["depth"]
|
|
|
|
|
|
|
|
|
|
return max_depth_event_id, current_max_depth
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
async def get_min_depth_of(self, event_ids: List[str]) -> Tuple[Optional[str], int]:
|
2021-06-22 09:02:53 +00:00
|
|
|
|
"""Returns the event ID and depth for the event that has the min depth from a set of event IDs
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
event_ids: The event IDs to calculate the max depth of.
|
|
|
|
|
"""
|
|
|
|
|
rows = await self.db_pool.simple_select_many_batch(
|
|
|
|
|
table="events",
|
|
|
|
|
column="event_id",
|
|
|
|
|
iterable=event_ids,
|
|
|
|
|
retcols=(
|
|
|
|
|
"event_id",
|
|
|
|
|
"depth",
|
|
|
|
|
),
|
|
|
|
|
desc="get_min_depth_of",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
if not rows:
|
|
|
|
|
return None, 0
|
|
|
|
|
else:
|
|
|
|
|
min_depth_event_id = ""
|
|
|
|
|
current_min_depth = MAX_DEPTH
|
|
|
|
|
for row in rows:
|
|
|
|
|
if row["depth"] < current_min_depth:
|
|
|
|
|
min_depth_event_id = row["event_id"]
|
|
|
|
|
current_min_depth = row["depth"]
|
|
|
|
|
|
|
|
|
|
return min_depth_event_id, current_min_depth
|
2019-01-25 17:19:31 +00:00
|
|
|
|
|
2020-08-28 11:54:27 +00:00
|
|
|
|
async def get_prev_events_for_room(self, room_id: str) -> List[str]:
|
2020-01-03 16:09:24 +00:00
|
|
|
|
"""
|
|
|
|
|
Gets a subset of the current forward extremities in the given room.
|
|
|
|
|
|
|
|
|
|
Limits the result to 10 extremities, so that we can avoid creating
|
|
|
|
|
events which refer to hundreds of prev_events.
|
|
|
|
|
|
|
|
|
|
Args:
|
2020-08-28 11:54:27 +00:00
|
|
|
|
room_id: room_id
|
2020-01-03 16:09:24 +00:00
|
|
|
|
|
|
|
|
|
Returns:
|
2020-08-28 11:54:27 +00:00
|
|
|
|
The event ids of the forward extremities.
|
2020-01-03 16:09:24 +00:00
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
2020-08-28 11:54:27 +00:00
|
|
|
|
return await self.db_pool.runInteraction(
|
2020-01-03 16:09:24 +00:00
|
|
|
|
"get_prev_events_for_room", self._get_prev_events_for_room_txn, room_id
|
|
|
|
|
)
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
def _get_prev_events_for_room_txn(
|
|
|
|
|
self, txn: LoggingTransaction, room_id: str
|
|
|
|
|
) -> List[str]:
|
2020-01-03 16:09:24 +00:00
|
|
|
|
# we just use the 10 newest events. Older events will become
|
|
|
|
|
# prev_events of future events.
|
|
|
|
|
|
|
|
|
|
sql = """
|
|
|
|
|
SELECT e.event_id FROM event_forward_extremities AS f
|
|
|
|
|
INNER JOIN events AS e USING (event_id)
|
|
|
|
|
WHERE f.room_id = ?
|
|
|
|
|
ORDER BY e.depth DESC
|
|
|
|
|
LIMIT 10
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
txn.execute(sql, (room_id,))
|
|
|
|
|
|
|
|
|
|
return [row[0] for row in txn]
|
|
|
|
|
|
2020-08-28 11:54:27 +00:00
|
|
|
|
async def get_rooms_with_many_extremities(
|
|
|
|
|
self, min_count: int, limit: int, room_id_filter: Iterable[str]
|
|
|
|
|
) -> List[str]:
|
2019-06-17 17:04:42 +00:00
|
|
|
|
"""Get the top rooms with at least N extremities.
|
|
|
|
|
|
|
|
|
|
Args:
|
2020-08-28 11:54:27 +00:00
|
|
|
|
min_count: The minimum number of extremities
|
|
|
|
|
limit: The maximum number of rooms to return.
|
|
|
|
|
room_id_filter: room_ids to exclude from the results
|
2019-06-17 17:04:42 +00:00
|
|
|
|
|
|
|
|
|
Returns:
|
2020-08-28 11:54:27 +00:00
|
|
|
|
At most `limit` room IDs that have at least `min_count` extremities,
|
|
|
|
|
sorted by extremity count.
|
2019-06-17 17:04:42 +00:00
|
|
|
|
"""
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
def _get_rooms_with_many_extremities_txn(txn: LoggingTransaction) -> List[str]:
|
2019-09-26 10:47:53 +00:00
|
|
|
|
where_clause = "1=1"
|
|
|
|
|
if room_id_filter:
|
|
|
|
|
where_clause = "room_id NOT IN (%s)" % (
|
|
|
|
|
",".join("?" for _ in room_id_filter),
|
|
|
|
|
)
|
|
|
|
|
|
2019-06-17 17:04:42 +00:00
|
|
|
|
sql = """
|
|
|
|
|
SELECT room_id FROM event_forward_extremities
|
2019-09-26 10:47:53 +00:00
|
|
|
|
WHERE %s
|
2019-06-17 17:04:42 +00:00
|
|
|
|
GROUP BY room_id
|
|
|
|
|
HAVING count(*) > ?
|
|
|
|
|
ORDER BY count(*) DESC
|
|
|
|
|
LIMIT ?
|
2019-09-26 10:47:53 +00:00
|
|
|
|
""" % (
|
|
|
|
|
where_clause,
|
|
|
|
|
)
|
2019-06-17 17:04:42 +00:00
|
|
|
|
|
2019-09-26 10:47:53 +00:00
|
|
|
|
query_args = list(itertools.chain(room_id_filter, [min_count, limit]))
|
|
|
|
|
txn.execute(sql, query_args)
|
2019-06-17 17:04:42 +00:00
|
|
|
|
return [room_id for room_id, in txn]
|
|
|
|
|
|
2020-08-28 11:54:27 +00:00
|
|
|
|
return await self.db_pool.runInteraction(
|
2019-06-17 17:04:42 +00:00
|
|
|
|
"get_rooms_with_many_extremities", _get_rooms_with_many_extremities_txn
|
|
|
|
|
)
|
|
|
|
|
|
2017-02-01 10:39:41 +00:00
|
|
|
|
@cached(max_entries=5000, iterable=True)
|
2023-02-10 23:29:00 +00:00
|
|
|
|
async def get_latest_event_ids_in_room(self, room_id: str) -> Sequence[str]:
|
2020-08-27 11:08:38 +00:00
|
|
|
|
return await self.db_pool.simple_select_onecol(
|
2015-05-01 09:17:19 +00:00
|
|
|
|
table="event_forward_extremities",
|
2019-04-03 09:07:29 +00:00
|
|
|
|
keyvalues={"room_id": room_id},
|
2015-05-01 09:17:19 +00:00
|
|
|
|
retcol="event_id",
|
2015-05-05 09:24:10 +00:00
|
|
|
|
desc="get_latest_event_ids_in_room",
|
2015-05-01 09:17:19 +00:00
|
|
|
|
)
|
|
|
|
|
|
2021-10-18 16:17:15 +00:00
|
|
|
|
async def get_min_depth(self, room_id: str) -> Optional[int]:
|
2020-08-28 11:54:27 +00:00
|
|
|
|
"""For the given room, get the minimum depth we have seen for it."""
|
|
|
|
|
return await self.db_pool.runInteraction(
|
2019-04-03 09:07:29 +00:00
|
|
|
|
"get_min_depth", self._get_min_depth_interaction, room_id
|
2014-10-31 10:47:34 +00:00
|
|
|
|
)
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
def _get_min_depth_interaction(
|
|
|
|
|
self, txn: LoggingTransaction, room_id: str
|
|
|
|
|
) -> Optional[int]:
|
2020-08-05 20:38:57 +00:00
|
|
|
|
min_depth = self.db_pool.simple_select_one_onecol_txn(
|
2014-10-28 16:42:35 +00:00
|
|
|
|
txn,
|
|
|
|
|
table="room_depth",
|
2014-11-10 13:46:44 +00:00
|
|
|
|
keyvalues={"room_id": room_id},
|
2014-10-28 16:42:35 +00:00
|
|
|
|
retcol="min_depth",
|
|
|
|
|
allow_none=True,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
return int(min_depth) if min_depth is not None else None
|
|
|
|
|
|
2023-04-06 16:42:39 +00:00
|
|
|
|
async def have_room_forward_extremities_changed_since(
|
|
|
|
|
self,
|
|
|
|
|
room_id: str,
|
|
|
|
|
stream_ordering: int,
|
|
|
|
|
) -> bool:
|
|
|
|
|
"""Check if the forward extremities in a room have changed since the
|
|
|
|
|
given stream ordering
|
|
|
|
|
|
|
|
|
|
Throws a StoreError if we have since purged the index for
|
|
|
|
|
stream_orderings from that point.
|
|
|
|
|
"""
|
2023-04-14 18:04:49 +00:00
|
|
|
|
assert self.stream_ordering_month_ago is not None
|
|
|
|
|
if stream_ordering <= self.stream_ordering_month_ago:
|
2023-04-06 16:42:39 +00:00
|
|
|
|
raise StoreError(400, f"stream_ordering too old {stream_ordering}")
|
|
|
|
|
|
|
|
|
|
sql = """
|
|
|
|
|
SELECT 1 FROM stream_ordering_to_exterm
|
|
|
|
|
WHERE stream_ordering > ? AND room_id = ?
|
|
|
|
|
LIMIT 1
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
def have_room_forward_extremities_changed_since_txn(
|
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
|
) -> bool:
|
|
|
|
|
txn.execute(sql, (stream_ordering, room_id))
|
|
|
|
|
return txn.fetchone() is not None
|
|
|
|
|
|
|
|
|
|
return await self.db_pool.runInteraction(
|
|
|
|
|
"have_room_forward_extremities_changed_since",
|
|
|
|
|
have_room_forward_extremities_changed_since_txn,
|
|
|
|
|
)
|
|
|
|
|
|
2022-09-07 11:03:32 +00:00
|
|
|
|
@cancellable
|
2021-03-17 13:20:08 +00:00
|
|
|
|
async def get_forward_extremities_for_room_at_stream_ordering(
|
2020-08-28 11:54:27 +00:00
|
|
|
|
self, room_id: str, stream_ordering: int
|
2023-02-10 23:29:00 +00:00
|
|
|
|
) -> Sequence[str]:
|
2017-02-14 13:59:50 +00:00
|
|
|
|
"""For a given room_id and stream_ordering, return the forward
|
|
|
|
|
extremeties of the room at that point in "time".
|
|
|
|
|
|
|
|
|
|
Throws a StoreError if we have since purged the index for
|
|
|
|
|
stream_orderings from that point.
|
|
|
|
|
|
|
|
|
|
Args:
|
2020-08-28 11:54:27 +00:00
|
|
|
|
room_id:
|
|
|
|
|
stream_ordering:
|
2017-02-14 13:59:50 +00:00
|
|
|
|
|
|
|
|
|
Returns:
|
2020-08-28 11:54:27 +00:00
|
|
|
|
A list of event_ids
|
2017-02-14 13:59:50 +00:00
|
|
|
|
"""
|
2016-09-15 13:27:15 +00:00
|
|
|
|
# We want to make the cache more effective, so we clamp to the last
|
|
|
|
|
# change before the given ordering.
|
2022-05-18 15:02:10 +00:00
|
|
|
|
last_change = self._events_stream_cache.get_max_pos_of_last_change(room_id) # type: ignore[attr-defined]
|
2016-09-15 16:34:59 +00:00
|
|
|
|
|
|
|
|
|
# We don't always have a full stream_to_exterm_id table, e.g. after
|
|
|
|
|
# the upgrade that introduced it, so we make sure we never ask for a
|
2017-02-14 13:59:50 +00:00
|
|
|
|
# stream_ordering from before a restart
|
2022-05-18 15:02:10 +00:00
|
|
|
|
last_change = max(self._stream_order_on_start, last_change) # type: ignore[attr-defined]
|
2016-09-15 16:34:59 +00:00
|
|
|
|
|
2017-02-14 13:59:50 +00:00
|
|
|
|
# provided the last_change is recent enough, we now clamp the requested
|
|
|
|
|
# stream_ordering to it.
|
2023-04-14 18:04:49 +00:00
|
|
|
|
assert self.stream_ordering_month_ago is not None
|
|
|
|
|
if last_change > self.stream_ordering_month_ago:
|
2016-09-15 16:34:59 +00:00
|
|
|
|
stream_ordering = min(last_change, stream_ordering)
|
2016-09-15 13:27:15 +00:00
|
|
|
|
|
2020-08-28 11:54:27 +00:00
|
|
|
|
return await self._get_forward_extremeties_for_room(room_id, stream_ordering)
|
2016-09-15 13:27:15 +00:00
|
|
|
|
|
|
|
|
|
@cached(max_entries=5000, num_args=2)
|
2022-05-18 15:02:10 +00:00
|
|
|
|
async def _get_forward_extremeties_for_room(
|
|
|
|
|
self, room_id: str, stream_ordering: int
|
2023-02-10 23:29:00 +00:00
|
|
|
|
) -> Sequence[str]:
|
2016-09-14 15:09:32 +00:00
|
|
|
|
"""For a given room_id and stream_ordering, return the forward
|
|
|
|
|
extremeties of the room at that point in "time".
|
|
|
|
|
|
|
|
|
|
Throws a StoreError if we have since purged the index for
|
|
|
|
|
stream_orderings from that point.
|
|
|
|
|
"""
|
2023-04-14 18:04:49 +00:00
|
|
|
|
assert self.stream_ordering_month_ago is not None
|
|
|
|
|
if stream_ordering <= self.stream_ordering_month_ago:
|
2020-09-14 09:16:41 +00:00
|
|
|
|
raise StoreError(400, "stream_ordering too old %s" % (stream_ordering,))
|
2016-09-14 15:09:32 +00:00
|
|
|
|
|
2019-04-03 09:07:29 +00:00
|
|
|
|
sql = """
|
2016-09-14 15:09:32 +00:00
|
|
|
|
SELECT event_id FROM stream_ordering_to_exterm
|
|
|
|
|
INNER JOIN (
|
|
|
|
|
SELECT room_id, MAX(stream_ordering) AS stream_ordering
|
|
|
|
|
FROM stream_ordering_to_exterm
|
2016-09-16 09:19:32 +00:00
|
|
|
|
WHERE stream_ordering <= ? GROUP BY room_id
|
2016-09-14 15:09:32 +00:00
|
|
|
|
) AS rms USING (room_id, stream_ordering)
|
|
|
|
|
WHERE room_id = ?
|
2019-04-03 09:07:29 +00:00
|
|
|
|
"""
|
2016-09-14 15:09:32 +00:00
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
def get_forward_extremeties_for_room_txn(txn: LoggingTransaction) -> List[str]:
|
2016-09-14 16:01:02 +00:00
|
|
|
|
txn.execute(sql, (stream_ordering, room_id))
|
2017-03-23 17:53:49 +00:00
|
|
|
|
return [event_id for event_id, in txn]
|
2016-09-14 15:09:32 +00:00
|
|
|
|
|
2023-04-06 16:42:39 +00:00
|
|
|
|
event_ids = await self.db_pool.runInteraction(
|
2019-04-03 09:07:29 +00:00
|
|
|
|
"get_forward_extremeties_for_room", get_forward_extremeties_for_room_txn
|
2016-09-14 15:09:32 +00:00
|
|
|
|
)
|
|
|
|
|
|
2023-04-06 16:42:39 +00:00
|
|
|
|
# If we didn't find any IDs, then we must have cleared out the
|
|
|
|
|
# associated `stream_ordering_to_exterm`.
|
|
|
|
|
if not event_ids:
|
|
|
|
|
raise StoreError(400, "stream_ordering too old %s" % (stream_ordering,))
|
|
|
|
|
|
|
|
|
|
return event_ids
|
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
def _get_connected_prev_event_backfill_results_txn(
|
|
|
|
|
self, txn: LoggingTransaction, event_id: str, limit: int
|
|
|
|
|
) -> List[BackfillQueueNavigationItem]:
|
|
|
|
|
"""
|
|
|
|
|
Find any events connected by prev_event the specified event_id.
|
2014-10-31 09:59:02 +00:00
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
Args:
|
|
|
|
|
txn: The database transaction to use
|
|
|
|
|
event_id: The event ID to navigate from
|
|
|
|
|
limit: Max number of event ID's to query for and return
|
2014-10-31 09:59:02 +00:00
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
Returns:
|
|
|
|
|
List of prev events that the backfill queue can process
|
|
|
|
|
"""
|
2021-07-28 15:46:37 +00:00
|
|
|
|
# Look for the prev_event_id connected to the given event_id
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
connected_prev_event_query = """
|
|
|
|
|
SELECT depth, stream_ordering, prev_event_id, events.type FROM event_edges
|
|
|
|
|
/* Get the depth and stream_ordering of the prev_event_id from the events table */
|
2021-07-28 15:46:37 +00:00
|
|
|
|
INNER JOIN events
|
|
|
|
|
ON prev_event_id = events.event_id
|
2022-03-28 18:20:14 +00:00
|
|
|
|
|
|
|
|
|
/* exclude outliers from the results (we don't have the state, so cannot
|
|
|
|
|
* verify if the requesting server can see them).
|
|
|
|
|
*/
|
|
|
|
|
WHERE NOT events.outlier
|
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
/* Look for an edge which matches the given event_id */
|
2022-03-28 18:20:14 +00:00
|
|
|
|
AND event_edges.event_id = ? AND NOT event_edges.is_state
|
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
/* Because we can have many events at the same depth,
|
|
|
|
|
* we want to also tie-break and sort on stream_ordering */
|
|
|
|
|
ORDER BY depth DESC, stream_ordering DESC
|
2021-07-28 15:46:37 +00:00
|
|
|
|
LIMIT ?
|
|
|
|
|
"""
|
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
txn.execute(
|
|
|
|
|
connected_prev_event_query,
|
2022-03-28 18:20:14 +00:00
|
|
|
|
(event_id, limit),
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
)
|
|
|
|
|
return [
|
|
|
|
|
BackfillQueueNavigationItem(
|
|
|
|
|
depth=row[0],
|
|
|
|
|
stream_ordering=row[1],
|
|
|
|
|
event_id=row[2],
|
|
|
|
|
type=row[3],
|
|
|
|
|
)
|
|
|
|
|
for row in txn
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
async def get_backfill_events(
|
2022-05-18 15:02:10 +00:00
|
|
|
|
self, room_id: str, seed_event_id_list: List[str], limit: int
|
|
|
|
|
) -> List[EventBase]:
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
"""Get a list of Events for a given topic that occurred before (and
|
|
|
|
|
including) the events in seed_event_id_list. Return a list of max size `limit`
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
room_id
|
|
|
|
|
seed_event_id_list
|
|
|
|
|
limit
|
2021-07-28 15:46:37 +00:00
|
|
|
|
"""
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
event_ids = await self.db_pool.runInteraction(
|
|
|
|
|
"get_backfill_events",
|
|
|
|
|
self._get_backfill_events,
|
|
|
|
|
room_id,
|
|
|
|
|
seed_event_id_list,
|
|
|
|
|
limit,
|
|
|
|
|
)
|
|
|
|
|
events = await self.get_events_as_list(event_ids)
|
|
|
|
|
return sorted(
|
2022-05-18 15:02:10 +00:00
|
|
|
|
# type-ignore: mypy doesn't like negating the Optional[int] stream_ordering.
|
|
|
|
|
# But it's never None, because these events were previously persisted to the DB.
|
|
|
|
|
events,
|
|
|
|
|
key=lambda e: (-e.depth, -e.internal_metadata.stream_ordering), # type: ignore[operator]
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
)
|
2021-07-28 15:46:37 +00:00
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
def _get_backfill_events(
|
|
|
|
|
self,
|
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
|
room_id: str,
|
|
|
|
|
seed_event_id_list: List[str],
|
|
|
|
|
limit: int,
|
|
|
|
|
) -> Set[str]:
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
"""
|
|
|
|
|
We want to make sure that we do a breadth-first, "depth" ordered search.
|
|
|
|
|
We also handle navigating historical branches of history connected by
|
|
|
|
|
insertion and batch events.
|
2021-07-28 15:46:37 +00:00
|
|
|
|
"""
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
logger.debug(
|
|
|
|
|
"_get_backfill_events(room_id=%s): seeding backfill with seed_event_id_list=%s limit=%s",
|
|
|
|
|
room_id,
|
|
|
|
|
seed_event_id_list,
|
|
|
|
|
limit,
|
|
|
|
|
)
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
event_id_results: Set[str] = set()
|
2014-10-31 09:59:02 +00:00
|
|
|
|
|
2021-07-28 15:46:37 +00:00
|
|
|
|
# In a PriorityQueue, the lowest valued entries are retrieved first.
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
# We're using depth as the priority in the queue and tie-break based on
|
|
|
|
|
# stream_ordering. Depth is lowest at the oldest-in-time message and
|
|
|
|
|
# highest and newest-in-time message. We add events to the queue with a
|
|
|
|
|
# negative depth so that we process the newest-in-time messages first
|
|
|
|
|
# going backwards in time. stream_ordering follows the same pattern.
|
2022-05-18 15:02:10 +00:00
|
|
|
|
queue: "PriorityQueue[Tuple[int, int, str, str]]" = PriorityQueue()
|
2014-10-31 09:59:02 +00:00
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
for seed_event_id in seed_event_id_list:
|
|
|
|
|
event_lookup_result = self.db_pool.simple_select_one_txn(
|
2015-05-21 14:46:07 +00:00
|
|
|
|
txn,
|
|
|
|
|
table="events",
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
keyvalues={"event_id": seed_event_id, "room_id": room_id},
|
|
|
|
|
retcols=(
|
|
|
|
|
"type",
|
|
|
|
|
"depth",
|
|
|
|
|
"stream_ordering",
|
|
|
|
|
),
|
2015-07-06 08:31:40 +00:00
|
|
|
|
allow_none=True,
|
2015-05-21 14:46:07 +00:00
|
|
|
|
)
|
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
if event_lookup_result is not None:
|
|
|
|
|
logger.debug(
|
|
|
|
|
"_get_backfill_events(room_id=%s): seed_event_id=%s depth=%s stream_ordering=%s type=%s",
|
|
|
|
|
room_id,
|
|
|
|
|
seed_event_id,
|
|
|
|
|
event_lookup_result["depth"],
|
|
|
|
|
event_lookup_result["stream_ordering"],
|
|
|
|
|
event_lookup_result["type"],
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
if event_lookup_result["depth"]:
|
|
|
|
|
queue.put(
|
|
|
|
|
(
|
|
|
|
|
-event_lookup_result["depth"],
|
|
|
|
|
-event_lookup_result["stream_ordering"],
|
|
|
|
|
seed_event_id,
|
|
|
|
|
event_lookup_result["type"],
|
|
|
|
|
)
|
|
|
|
|
)
|
2014-10-31 09:59:02 +00:00
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
while not queue.empty() and len(event_id_results) < limit:
|
2015-05-21 14:37:43 +00:00
|
|
|
|
try:
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
_, _, event_id, event_type = queue.get_nowait()
|
2015-05-21 14:37:43 +00:00
|
|
|
|
except Empty:
|
|
|
|
|
break
|
2014-10-31 09:59:02 +00:00
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
if event_id in event_id_results:
|
2015-05-21 14:48:56 +00:00
|
|
|
|
continue
|
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
event_id_results.add(event_id)
|
2015-05-20 11:57:00 +00:00
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
# Now we just look up the DAG by prev_events as normal
|
|
|
|
|
connected_prev_event_backfill_results = (
|
|
|
|
|
self._get_connected_prev_event_backfill_results_txn(
|
|
|
|
|
txn, event_id, limit - len(event_id_results)
|
2021-07-28 15:46:37 +00:00
|
|
|
|
)
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
)
|
2021-07-28 15:46:37 +00:00
|
|
|
|
logger.debug(
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
"_get_backfill_events(room_id=%s): connected_prev_event_backfill_results=%s",
|
|
|
|
|
room_id,
|
|
|
|
|
connected_prev_event_backfill_results,
|
2021-07-28 15:46:37 +00:00
|
|
|
|
)
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
for (
|
|
|
|
|
connected_prev_event_backfill_item
|
|
|
|
|
) in connected_prev_event_backfill_results:
|
|
|
|
|
if connected_prev_event_backfill_item.event_id not in event_id_results:
|
|
|
|
|
queue.put(
|
|
|
|
|
(
|
|
|
|
|
-connected_prev_event_backfill_item.depth,
|
|
|
|
|
-connected_prev_event_backfill_item.stream_ordering,
|
|
|
|
|
connected_prev_event_backfill_item.event_id,
|
|
|
|
|
connected_prev_event_backfill_item.type,
|
|
|
|
|
)
|
|
|
|
|
)
|
2015-05-20 11:57:00 +00:00
|
|
|
|
|
Fix historical messages backfilling in random order on remote homeservers (MSC2716) (#11114)
Fix https://github.com/matrix-org/synapse/issues/11091
Fix https://github.com/matrix-org/synapse/issues/10764 (side-stepping the issue because we no longer have to deal with `fake_prev_event_id`)
1. Made the `/backfill` response return messages in `(depth, stream_ordering)` order (previously only sorted by `depth`)
- Technically, it shouldn't really matter how `/backfill` returns things but I'm just trying to make the `stream_ordering` a little more consistent from the origin to the remote homeservers in order to get the order of messages from `/messages` consistent ([sorted by `(topological_ordering, stream_ordering)`](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)).
- Even now that we return backfilled messages in order, it still doesn't guarantee the same `stream_ordering` (and more importantly the [`/messages` order](https://github.com/matrix-org/synapse/blob/develop/docs/development/room-dag-concepts.md#depth-and-stream-ordering)) on the other server. For example, if a room has a bunch of history imported and someone visits a permalink to a historical message back in time, their homeserver will skip over the historical messages in between and insert the permalink as the next message in the `stream_order` and totally throw off the sort.
- This will be even more the case when we add the [MSC3030 jump to date API endpoint](https://github.com/matrix-org/matrix-doc/pull/3030) so the static archives can navigate and jump to a certain date.
- We're solving this in the future by switching to [online topological ordering](https://github.com/matrix-org/gomatrixserverlib/issues/187) and [chunking](https://github.com/matrix-org/synapse/issues/3785) which by its nature will apply retroactively to fix any inconsistencies introduced by people permalinking
2. As we're navigating `prev_events` to return in `/backfill`, we order by `depth` first (newest -> oldest) and now also tie-break based on the `stream_ordering` (newest -> oldest). This is technically important because MSC2716 inserts a bunch of historical messages at the same `depth` so it's best to be prescriptive about which ones we should process first. In reality, I think the code already looped over the historical messages as expected because the database is already in order.
3. Making the historical state chain and historical event chain float on their own by having no `prev_events` instead of a fake `prev_event` which caused backfill to get clogged with an unresolvable event. Fixes https://github.com/matrix-org/synapse/issues/11091 and https://github.com/matrix-org/synapse/issues/10764
4. We no longer find connected insertion events by finding a potential `prev_event` connection to the current event we're iterating over. We now solely rely on marker events which when processed, add the insertion event as an extremity and the federating homeserver can ask about it when time calls.
- Related discussion, https://github.com/matrix-org/synapse/pull/11114#discussion_r741514793
Before | After
--- | ---
![](https://user-images.githubusercontent.com/558581/139218681-b465c862-5c49-4702-a59e-466733b0cf45.png) | ![](https://user-images.githubusercontent.com/558581/146453159-a1609e0a-8324-439d-ae44-e4bce43ac6d1.png)
#### Why aren't we sorting topologically when receiving backfill events?
> The main reason we're going to opt to not sort topologically when receiving backfill events is because it's probably best to do whatever is easiest to make it just work. People will probably have opinions once they look at [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) which could change whatever implementation anyway.
>
> As mentioned, ideally we would do this but code necessary to make the fake edges but it gets confusing and gives an impression of “just whyyyy” (feels icky). This problem also dissolves with online topological ordering.
>
> -- https://github.com/matrix-org/synapse/pull/11114#discussion_r741517138
See https://github.com/matrix-org/synapse/pull/11114#discussion_r739610091 for the technical difficulties
2022-02-07 21:54:13 +00:00
|
|
|
|
return event_id_results
|
2015-02-19 17:24:14 +00:00
|
|
|
|
|
2022-09-14 18:57:50 +00:00
|
|
|
|
@trace
|
|
|
|
|
async def record_event_failed_pull_attempt(
|
|
|
|
|
self, room_id: str, event_id: str, cause: str
|
|
|
|
|
) -> None:
|
|
|
|
|
"""
|
|
|
|
|
Record when we fail to pull an event over federation.
|
|
|
|
|
|
|
|
|
|
This information allows us to be more intelligent when we decide to
|
|
|
|
|
retry (we don't need to fail over and over) and we can process that
|
|
|
|
|
event in the background so we don't block on it each time.
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
room_id: The room where the event failed to pull from
|
|
|
|
|
event_id: The event that failed to be fetched or processed
|
|
|
|
|
cause: The error message or reason that we failed to pull the event
|
|
|
|
|
"""
|
2022-10-15 05:36:49 +00:00
|
|
|
|
logger.debug(
|
|
|
|
|
"record_event_failed_pull_attempt room_id=%s, event_id=%s, cause=%s",
|
|
|
|
|
room_id,
|
|
|
|
|
event_id,
|
|
|
|
|
cause,
|
|
|
|
|
)
|
2022-09-14 18:57:50 +00:00
|
|
|
|
await self.db_pool.runInteraction(
|
|
|
|
|
"record_event_failed_pull_attempt",
|
|
|
|
|
self._record_event_failed_pull_attempt_upsert_txn,
|
|
|
|
|
room_id,
|
|
|
|
|
event_id,
|
|
|
|
|
cause,
|
|
|
|
|
db_autocommit=True, # Safe as it's a single upsert
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def _record_event_failed_pull_attempt_upsert_txn(
|
|
|
|
|
self,
|
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
|
room_id: str,
|
|
|
|
|
event_id: str,
|
|
|
|
|
cause: str,
|
|
|
|
|
) -> None:
|
|
|
|
|
sql = """
|
|
|
|
|
INSERT INTO event_failed_pull_attempts (
|
|
|
|
|
room_id, event_id, num_attempts, last_attempt_ts, last_cause
|
|
|
|
|
)
|
|
|
|
|
VALUES (?, ?, ?, ?, ?)
|
|
|
|
|
ON CONFLICT (room_id, event_id) DO UPDATE SET
|
|
|
|
|
num_attempts=event_failed_pull_attempts.num_attempts + 1,
|
|
|
|
|
last_attempt_ts=EXCLUDED.last_attempt_ts,
|
|
|
|
|
last_cause=EXCLUDED.last_cause;
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
txn.execute(sql, (room_id, event_id, 1, self._clock.time_msec(), cause))
|
|
|
|
|
|
2023-05-25 04:22:24 +00:00
|
|
|
|
@trace
|
|
|
|
|
async def get_event_ids_with_failed_pull_attempts(
|
|
|
|
|
self, event_ids: StrCollection
|
|
|
|
|
) -> Set[str]:
|
|
|
|
|
"""
|
|
|
|
|
Filter the given list of `event_ids` and return events which have any failed
|
|
|
|
|
pull attempts.
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
event_ids: A list of events to filter down.
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
A filtered down list of `event_ids` that have previous failed pull attempts.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
rows = await self.db_pool.simple_select_many_batch(
|
|
|
|
|
table="event_failed_pull_attempts",
|
|
|
|
|
column="event_id",
|
|
|
|
|
iterable=event_ids,
|
|
|
|
|
keyvalues={},
|
|
|
|
|
retcols=("event_id",),
|
|
|
|
|
desc="get_event_ids_with_failed_pull_attempts",
|
|
|
|
|
)
|
|
|
|
|
event_ids_with_failed_pull_attempts: Set[str] = {
|
|
|
|
|
row["event_id"] for row in rows
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return event_ids_with_failed_pull_attempts
|
|
|
|
|
|
2022-10-15 05:36:49 +00:00
|
|
|
|
@trace
|
|
|
|
|
async def get_event_ids_to_not_pull_from_backoff(
|
|
|
|
|
self,
|
|
|
|
|
room_id: str,
|
|
|
|
|
event_ids: Collection[str],
|
2023-03-30 12:36:41 +00:00
|
|
|
|
) -> Dict[str, int]:
|
2022-10-15 05:36:49 +00:00
|
|
|
|
"""
|
|
|
|
|
Filter down the events to ones that we've failed to pull before recently. Uses
|
|
|
|
|
exponential backoff.
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
room_id: The room that the events belong to
|
|
|
|
|
event_ids: A list of events to filter down
|
|
|
|
|
|
|
|
|
|
Returns:
|
2023-03-30 12:36:41 +00:00
|
|
|
|
A dictionary of event_ids that should not be attempted to be pulled and the
|
|
|
|
|
next timestamp at which we may try pulling them again.
|
2022-10-15 05:36:49 +00:00
|
|
|
|
"""
|
|
|
|
|
event_failed_pull_attempts = await self.db_pool.simple_select_many_batch(
|
|
|
|
|
table="event_failed_pull_attempts",
|
|
|
|
|
column="event_id",
|
|
|
|
|
iterable=event_ids,
|
|
|
|
|
keyvalues={},
|
|
|
|
|
retcols=(
|
|
|
|
|
"event_id",
|
|
|
|
|
"last_attempt_ts",
|
|
|
|
|
"num_attempts",
|
|
|
|
|
),
|
|
|
|
|
desc="get_event_ids_to_not_pull_from_backoff",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
current_time = self._clock.time_msec()
|
2023-03-30 12:36:41 +00:00
|
|
|
|
|
|
|
|
|
event_ids_with_backoff = {}
|
|
|
|
|
for event_failed_pull_attempt in event_failed_pull_attempts:
|
|
|
|
|
event_id = event_failed_pull_attempt["event_id"]
|
2022-10-15 05:36:49 +00:00
|
|
|
|
# Exponential back-off (up to the upper bound) so we don't try to
|
|
|
|
|
# pull the same event over and over. ex. 2hr, 4hr, 8hr, 16hr, etc.
|
2023-03-30 12:36:41 +00:00
|
|
|
|
backoff_end_time = (
|
|
|
|
|
event_failed_pull_attempt["last_attempt_ts"]
|
|
|
|
|
+ (
|
|
|
|
|
2
|
|
|
|
|
** min(
|
|
|
|
|
event_failed_pull_attempt["num_attempts"],
|
|
|
|
|
BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS,
|
|
|
|
|
)
|
2022-10-15 05:36:49 +00:00
|
|
|
|
)
|
2023-03-30 12:36:41 +00:00
|
|
|
|
* BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS
|
2022-10-15 05:36:49 +00:00
|
|
|
|
)
|
2023-03-30 12:36:41 +00:00
|
|
|
|
|
|
|
|
|
if current_time < backoff_end_time: # `backoff_end_time` is exclusive
|
|
|
|
|
event_ids_with_backoff[event_id] = backoff_end_time
|
|
|
|
|
|
|
|
|
|
return event_ids_with_backoff
|
2022-10-15 05:36:49 +00:00
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
async def get_missing_events(
|
|
|
|
|
self,
|
|
|
|
|
room_id: str,
|
|
|
|
|
earliest_events: List[str],
|
|
|
|
|
latest_events: List[str],
|
|
|
|
|
limit: int,
|
|
|
|
|
) -> List[EventBase]:
|
2020-08-11 21:21:13 +00:00
|
|
|
|
ids = await self.db_pool.runInteraction(
|
2015-02-19 17:24:14 +00:00
|
|
|
|
"get_missing_events",
|
|
|
|
|
self._get_missing_events,
|
2019-04-03 09:07:29 +00:00
|
|
|
|
room_id,
|
|
|
|
|
earliest_events,
|
|
|
|
|
latest_events,
|
|
|
|
|
limit,
|
2015-02-19 17:24:14 +00:00
|
|
|
|
)
|
2020-08-18 20:20:49 +00:00
|
|
|
|
return await self.get_events_as_list(ids)
|
2015-05-14 12:45:48 +00:00
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
def _get_missing_events(
|
|
|
|
|
self,
|
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
|
room_id: str,
|
|
|
|
|
earliest_events: List[str],
|
|
|
|
|
latest_events: List[str],
|
|
|
|
|
limit: int,
|
|
|
|
|
) -> List[str]:
|
2018-10-16 19:37:16 +00:00
|
|
|
|
seen_events = set(earliest_events)
|
|
|
|
|
front = set(latest_events) - seen_events
|
2022-05-18 15:02:10 +00:00
|
|
|
|
event_results: List[str] = []
|
2015-02-19 17:24:14 +00:00
|
|
|
|
|
|
|
|
|
query = (
|
|
|
|
|
"SELECT prev_event_id FROM event_edges "
|
2022-05-31 12:51:49 +00:00
|
|
|
|
"WHERE event_id = ? AND NOT is_state "
|
2015-02-19 17:24:14 +00:00
|
|
|
|
"LIMIT ?"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
while front and len(event_results) < limit:
|
|
|
|
|
new_front = set()
|
|
|
|
|
for event_id in front:
|
2022-05-31 12:51:49 +00:00
|
|
|
|
txn.execute(query, (event_id, limit - len(event_results)))
|
2020-02-21 12:15:07 +00:00
|
|
|
|
new_results = {t[0] for t in txn} - seen_events
|
2015-02-19 17:24:14 +00:00
|
|
|
|
|
2018-10-16 19:37:16 +00:00
|
|
|
|
new_front |= new_results
|
|
|
|
|
seen_events |= new_results
|
|
|
|
|
event_results.extend(new_results)
|
2015-02-19 17:24:14 +00:00
|
|
|
|
|
|
|
|
|
front = new_front
|
|
|
|
|
|
2018-10-16 19:37:16 +00:00
|
|
|
|
# we built the list working backwards from latest_events; we now need to
|
|
|
|
|
# reverse it so that the events are approximately chronological.
|
|
|
|
|
event_results.reverse()
|
2015-05-14 12:45:48 +00:00
|
|
|
|
return event_results
|
2015-03-18 11:19:47 +00:00
|
|
|
|
|
2022-08-16 17:39:40 +00:00
|
|
|
|
@trace
|
|
|
|
|
@tag_args
|
2022-04-26 09:27:11 +00:00
|
|
|
|
async def get_successor_events(self, event_id: str) -> List[str]:
|
|
|
|
|
"""Fetch all events that have the given event as a prev event
|
2019-02-20 16:54:35 +00:00
|
|
|
|
|
|
|
|
|
Args:
|
2022-04-26 09:27:11 +00:00
|
|
|
|
event_id: The event to search for as a prev_event.
|
2019-02-20 16:54:35 +00:00
|
|
|
|
"""
|
2022-04-26 09:27:11 +00:00
|
|
|
|
return await self.db_pool.simple_select_onecol(
|
2019-02-20 16:54:35 +00:00
|
|
|
|
table="event_edges",
|
2022-04-26 09:27:11 +00:00
|
|
|
|
keyvalues={"prev_event_id": event_id},
|
|
|
|
|
retcol="event_id",
|
2019-04-03 09:07:29 +00:00
|
|
|
|
desc="get_successor_events",
|
2019-02-20 16:54:35 +00:00
|
|
|
|
)
|
|
|
|
|
|
2020-10-09 11:37:51 +00:00
|
|
|
|
@wrap_as_background_process("delete_old_forward_extrem_cache")
|
|
|
|
|
async def _delete_old_forward_extrem_cache(self) -> None:
|
2022-05-18 15:02:10 +00:00
|
|
|
|
def _delete_old_forward_extrem_cache_txn(txn: LoggingTransaction) -> None:
|
2020-10-09 11:37:51 +00:00
|
|
|
|
sql = """
|
|
|
|
|
DELETE FROM stream_ordering_to_exterm
|
2023-04-06 16:42:39 +00:00
|
|
|
|
WHERE stream_ordering < ?
|
2020-10-09 11:37:51 +00:00
|
|
|
|
"""
|
2023-04-14 18:04:49 +00:00
|
|
|
|
txn.execute(sql, (self.stream_ordering_month_ago,))
|
2020-10-09 11:37:51 +00:00
|
|
|
|
|
|
|
|
|
await self.db_pool.runInteraction(
|
|
|
|
|
"_delete_old_forward_extrem_cache",
|
|
|
|
|
_delete_old_forward_extrem_cache_txn,
|
|
|
|
|
)
|
|
|
|
|
|
2021-06-29 18:55:22 +00:00
|
|
|
|
async def insert_received_event_to_staging(
|
|
|
|
|
self, origin: str, event: EventBase
|
|
|
|
|
) -> None:
|
|
|
|
|
"""Insert a newly received event from federation into the staging area."""
|
|
|
|
|
|
|
|
|
|
# We use an upsert here to handle the case where we see the same event
|
|
|
|
|
# from the same server multiple times.
|
|
|
|
|
await self.db_pool.simple_upsert(
|
|
|
|
|
table="federation_inbound_events_staging",
|
|
|
|
|
keyvalues={
|
|
|
|
|
"origin": origin,
|
|
|
|
|
"event_id": event.event_id,
|
|
|
|
|
},
|
|
|
|
|
values={},
|
|
|
|
|
insertion_values={
|
|
|
|
|
"room_id": event.room_id,
|
|
|
|
|
"received_ts": self._clock.time_msec(),
|
|
|
|
|
"event_json": json_encoder.encode(event.get_dict()),
|
|
|
|
|
"internal_metadata": json_encoder.encode(
|
|
|
|
|
event.internal_metadata.get_dict()
|
|
|
|
|
),
|
|
|
|
|
},
|
|
|
|
|
desc="insert_received_event_to_staging",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
async def remove_received_event_from_staging(
|
|
|
|
|
self,
|
|
|
|
|
origin: str,
|
|
|
|
|
event_id: str,
|
2021-06-30 11:07:16 +00:00
|
|
|
|
) -> Optional[int]:
|
|
|
|
|
"""Remove the given event from the staging area.
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
The received_ts of the row that was deleted, if any.
|
|
|
|
|
"""
|
|
|
|
|
if self.db_pool.engine.supports_returning:
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
def _remove_received_event_from_staging_txn(
|
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
|
) -> Optional[int]:
|
2021-06-30 11:07:16 +00:00
|
|
|
|
sql = """
|
|
|
|
|
DELETE FROM federation_inbound_events_staging
|
|
|
|
|
WHERE origin = ? AND event_id = ?
|
|
|
|
|
RETURNING received_ts
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
txn.execute(sql, (origin, event_id))
|
2022-05-18 15:02:10 +00:00
|
|
|
|
row = cast(Optional[Tuple[int]], txn.fetchone())
|
2021-06-30 11:07:16 +00:00
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
if row is None:
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
return row[0]
|
|
|
|
|
|
|
|
|
|
return await self.db_pool.runInteraction(
|
2021-06-30 11:07:16 +00:00
|
|
|
|
"remove_received_event_from_staging",
|
|
|
|
|
_remove_received_event_from_staging_txn,
|
|
|
|
|
db_autocommit=True,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
def _remove_received_event_from_staging_txn(
|
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
|
) -> Optional[int]:
|
2021-06-30 11:07:16 +00:00
|
|
|
|
received_ts = self.db_pool.simple_select_one_onecol_txn(
|
|
|
|
|
txn,
|
|
|
|
|
table="federation_inbound_events_staging",
|
|
|
|
|
keyvalues={
|
|
|
|
|
"origin": origin,
|
|
|
|
|
"event_id": event_id,
|
|
|
|
|
},
|
|
|
|
|
retcol="received_ts",
|
|
|
|
|
allow_none=True,
|
|
|
|
|
)
|
|
|
|
|
self.db_pool.simple_delete_txn(
|
|
|
|
|
txn,
|
|
|
|
|
table="federation_inbound_events_staging",
|
|
|
|
|
keyvalues={
|
|
|
|
|
"origin": origin,
|
|
|
|
|
"event_id": event_id,
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
return received_ts
|
|
|
|
|
|
|
|
|
|
return await self.db_pool.runInteraction(
|
|
|
|
|
"remove_received_event_from_staging",
|
|
|
|
|
_remove_received_event_from_staging_txn,
|
|
|
|
|
)
|
2021-06-29 18:55:22 +00:00
|
|
|
|
|
|
|
|
|
async def get_next_staged_event_id_for_room(
|
|
|
|
|
self,
|
|
|
|
|
room_id: str,
|
|
|
|
|
) -> Optional[Tuple[str, str]]:
|
2022-09-23 19:01:29 +00:00
|
|
|
|
"""
|
|
|
|
|
Get the next event ID in the staging area for the given room.
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
Tuple of the `origin` and `event_id`
|
|
|
|
|
"""
|
2021-06-29 18:55:22 +00:00
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
def _get_next_staged_event_id_for_room_txn(
|
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
|
) -> Optional[Tuple[str, str]]:
|
2021-06-29 18:55:22 +00:00
|
|
|
|
sql = """
|
|
|
|
|
SELECT origin, event_id
|
|
|
|
|
FROM federation_inbound_events_staging
|
|
|
|
|
WHERE room_id = ?
|
|
|
|
|
ORDER BY received_ts ASC
|
|
|
|
|
LIMIT 1
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
txn.execute(sql, (room_id,))
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
return cast(Optional[Tuple[str, str]], txn.fetchone())
|
2021-06-29 18:55:22 +00:00
|
|
|
|
|
|
|
|
|
return await self.db_pool.runInteraction(
|
|
|
|
|
"get_next_staged_event_id_for_room", _get_next_staged_event_id_for_room_txn
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
async def get_next_staged_event_for_room(
|
|
|
|
|
self,
|
|
|
|
|
room_id: str,
|
|
|
|
|
room_version: RoomVersion,
|
|
|
|
|
) -> Optional[Tuple[str, EventBase]]:
|
|
|
|
|
"""Get the next event in the staging area for the given room."""
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
def _get_next_staged_event_for_room_txn(
|
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
|
) -> Optional[Tuple[str, str, str]]:
|
2021-06-29 18:55:22 +00:00
|
|
|
|
sql = """
|
|
|
|
|
SELECT event_json, internal_metadata, origin
|
|
|
|
|
FROM federation_inbound_events_staging
|
|
|
|
|
WHERE room_id = ?
|
|
|
|
|
ORDER BY received_ts ASC
|
|
|
|
|
LIMIT 1
|
|
|
|
|
"""
|
|
|
|
|
txn.execute(sql, (room_id,))
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
return cast(Optional[Tuple[str, str, str]], txn.fetchone())
|
2021-06-29 18:55:22 +00:00
|
|
|
|
|
|
|
|
|
row = await self.db_pool.runInteraction(
|
|
|
|
|
"get_next_staged_event_for_room", _get_next_staged_event_for_room_txn
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
if not row:
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
event_d = db_to_json(row[0])
|
|
|
|
|
internal_metadata_d = db_to_json(row[1])
|
|
|
|
|
origin = row[2]
|
|
|
|
|
|
|
|
|
|
event = make_event_from_dict(
|
|
|
|
|
event_dict=event_d,
|
|
|
|
|
room_version=room_version,
|
|
|
|
|
internal_metadata_dict=internal_metadata_d,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
return origin, event
|
|
|
|
|
|
2021-08-02 13:37:25 +00:00
|
|
|
|
async def prune_staged_events_in_room(
|
|
|
|
|
self,
|
|
|
|
|
room_id: str,
|
|
|
|
|
room_version: RoomVersion,
|
|
|
|
|
) -> bool:
|
|
|
|
|
"""Checks if there are lots of staged events for the room, and if so
|
|
|
|
|
prune them down.
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
Whether any events were pruned
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
# First check the size of the queue.
|
|
|
|
|
count = await self.db_pool.simple_select_one_onecol(
|
|
|
|
|
table="federation_inbound_events_staging",
|
|
|
|
|
keyvalues={"room_id": room_id},
|
2021-12-14 12:34:30 +00:00
|
|
|
|
retcol="COUNT(*)",
|
2021-08-02 13:37:25 +00:00
|
|
|
|
desc="prune_staged_events_in_room_count",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
if count < 100:
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
# If the queue is too large, then we want clear the entire queue,
|
|
|
|
|
# keeping only the forward extremities (i.e. the events not referenced
|
|
|
|
|
# by other events in the queue). We do this so that we can always
|
|
|
|
|
# backpaginate in all the events we have dropped.
|
|
|
|
|
rows = await self.db_pool.simple_select_list(
|
|
|
|
|
table="federation_inbound_events_staging",
|
|
|
|
|
keyvalues={"room_id": room_id},
|
|
|
|
|
retcols=("event_id", "event_json"),
|
|
|
|
|
desc="prune_staged_events_in_room_fetch",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Find the set of events referenced by those in the queue, as well as
|
|
|
|
|
# collecting all the event IDs in the queue.
|
|
|
|
|
referenced_events: Set[str] = set()
|
|
|
|
|
seen_events: Set[str] = set()
|
|
|
|
|
for row in rows:
|
|
|
|
|
event_id = row["event_id"]
|
|
|
|
|
seen_events.add(event_id)
|
|
|
|
|
event_d = db_to_json(row["event_json"])
|
|
|
|
|
|
|
|
|
|
# We don't bother parsing the dicts into full blown event objects,
|
|
|
|
|
# as that is needlessly expensive.
|
|
|
|
|
|
|
|
|
|
# We haven't checked that the `prev_events` have the right format
|
|
|
|
|
# yet, so we check as we go.
|
|
|
|
|
prev_events = event_d.get("prev_events", [])
|
|
|
|
|
if not isinstance(prev_events, list):
|
|
|
|
|
logger.info("Invalid prev_events for %s", event_id)
|
|
|
|
|
continue
|
|
|
|
|
|
2022-09-07 10:08:20 +00:00
|
|
|
|
if room_version.event_format == EventFormatVersions.ROOM_V1_V2:
|
2021-08-02 13:37:25 +00:00
|
|
|
|
for prev_event_tuple in prev_events:
|
2022-01-24 12:20:01 +00:00
|
|
|
|
if (
|
|
|
|
|
not isinstance(prev_event_tuple, list)
|
|
|
|
|
or len(prev_event_tuple) != 2
|
|
|
|
|
):
|
2021-08-02 13:37:25 +00:00
|
|
|
|
logger.info("Invalid prev_events for %s", event_id)
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
prev_event_id = prev_event_tuple[0]
|
|
|
|
|
if not isinstance(prev_event_id, str):
|
|
|
|
|
logger.info("Invalid prev_events for %s", event_id)
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
referenced_events.add(prev_event_id)
|
|
|
|
|
else:
|
|
|
|
|
for prev_event_id in prev_events:
|
|
|
|
|
if not isinstance(prev_event_id, str):
|
|
|
|
|
logger.info("Invalid prev_events for %s", event_id)
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
referenced_events.add(prev_event_id)
|
|
|
|
|
|
|
|
|
|
to_delete = referenced_events & seen_events
|
|
|
|
|
if not to_delete:
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
pdus_pruned_from_federation_queue.inc(len(to_delete))
|
|
|
|
|
logger.info(
|
|
|
|
|
"Pruning %d events in room %s from federation queue",
|
|
|
|
|
len(to_delete),
|
|
|
|
|
room_id,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
await self.db_pool.simple_delete_many(
|
|
|
|
|
table="federation_inbound_events_staging",
|
|
|
|
|
keyvalues={"room_id": room_id},
|
|
|
|
|
iterable=to_delete,
|
|
|
|
|
column="event_id",
|
|
|
|
|
desc="prune_staged_events_in_room_delete",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
2021-07-06 12:02:37 +00:00
|
|
|
|
async def get_all_rooms_with_staged_incoming_events(self) -> List[str]:
|
|
|
|
|
"""Get the room IDs of all events currently staged."""
|
|
|
|
|
return await self.db_pool.simple_select_onecol(
|
|
|
|
|
table="federation_inbound_events_staging",
|
|
|
|
|
keyvalues={},
|
|
|
|
|
retcol="DISTINCT room_id",
|
|
|
|
|
desc="get_all_rooms_with_staged_incoming_events",
|
|
|
|
|
)
|
|
|
|
|
|
2021-07-01 09:18:25 +00:00
|
|
|
|
@wrap_as_background_process("_get_stats_for_federation_staging")
|
2022-05-18 15:02:10 +00:00
|
|
|
|
async def _get_stats_for_federation_staging(self) -> None:
|
2021-07-01 09:18:25 +00:00
|
|
|
|
"""Update the prometheus metrics for the inbound federation staging area."""
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
def _get_stats_for_federation_staging_txn(
|
|
|
|
|
txn: LoggingTransaction,
|
|
|
|
|
) -> Tuple[int, int]:
|
2021-12-14 12:34:30 +00:00
|
|
|
|
txn.execute("SELECT count(*) FROM federation_inbound_events_staging")
|
2022-05-18 15:02:10 +00:00
|
|
|
|
(count,) = cast(Tuple[int], txn.fetchone())
|
2021-07-01 09:18:25 +00:00
|
|
|
|
|
|
|
|
|
txn.execute(
|
2021-07-27 17:01:04 +00:00
|
|
|
|
"SELECT min(received_ts) FROM federation_inbound_events_staging"
|
2021-07-01 09:18:25 +00:00
|
|
|
|
)
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
(received_ts,) = cast(Tuple[Optional[int]], txn.fetchone())
|
2021-07-13 10:33:15 +00:00
|
|
|
|
|
2021-07-27 17:01:04 +00:00
|
|
|
|
# If there is nothing in the staging area default it to 0.
|
|
|
|
|
age = 0
|
|
|
|
|
if received_ts is not None:
|
|
|
|
|
age = self._clock.time_msec() - received_ts
|
2021-07-01 09:18:25 +00:00
|
|
|
|
|
|
|
|
|
return count, age
|
|
|
|
|
|
|
|
|
|
count, age = await self.db_pool.runInteraction(
|
|
|
|
|
"_get_stats_for_federation_staging", _get_stats_for_federation_staging_txn
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
number_pdus_in_federation_queue.set(count)
|
|
|
|
|
oldest_pdu_in_federation_staging.set(age)
|
|
|
|
|
|
2018-03-01 14:16:02 +00:00
|
|
|
|
|
|
|
|
|
class EventFederationStore(EventFederationWorkerStore):
|
|
|
|
|
"""Responsible for storing and serving up the various graphs associated
|
|
|
|
|
with an event. Including the main event graph and the auth chains for an
|
|
|
|
|
event.
|
|
|
|
|
|
|
|
|
|
Also has methods for getting the front (latest) and back (oldest) edges
|
|
|
|
|
of the event graphs. These are used to generate the parents for new events
|
|
|
|
|
and backfilling from another server respectively.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
EVENT_AUTH_STATE_ONLY = "event_auth_state_only"
|
|
|
|
|
|
2021-12-13 17:05:00 +00:00
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
database: DatabasePool,
|
|
|
|
|
db_conn: LoggingDatabaseConnection,
|
|
|
|
|
hs: "HomeServer",
|
|
|
|
|
):
|
2020-09-18 13:56:44 +00:00
|
|
|
|
super().__init__(database, db_conn, hs)
|
2018-03-01 14:16:02 +00:00
|
|
|
|
|
2020-08-05 20:38:57 +00:00
|
|
|
|
self.db_pool.updates.register_background_update_handler(
|
2019-04-03 09:07:29 +00:00
|
|
|
|
self.EVENT_AUTH_STATE_ONLY, self._background_delete_non_state_event_auth
|
2018-03-01 14:16:02 +00:00
|
|
|
|
)
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
async def clean_room_for_join(self, room_id: str) -> None:
|
|
|
|
|
await self.db_pool.runInteraction(
|
2019-04-03 09:07:29 +00:00
|
|
|
|
"clean_room_for_join", self._clean_room_for_join_txn, room_id
|
2015-03-18 11:19:47 +00:00
|
|
|
|
)
|
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
def _clean_room_for_join_txn(self, txn: LoggingTransaction, room_id: str) -> None:
|
2015-03-18 11:19:47 +00:00
|
|
|
|
query = "DELETE FROM event_forward_extremities WHERE room_id = ?"
|
|
|
|
|
|
|
|
|
|
txn.execute(query, (room_id,))
|
2015-08-07 10:52:21 +00:00
|
|
|
|
txn.call_after(self.get_latest_event_ids_in_room.invalidate, (room_id,))
|
2017-05-24 13:58:13 +00:00
|
|
|
|
|
2022-05-18 15:02:10 +00:00
|
|
|
|
async def _background_delete_non_state_event_auth(
|
|
|
|
|
self, progress: JsonDict, batch_size: int
|
|
|
|
|
) -> int:
|
|
|
|
|
def delete_event_auth(txn: LoggingTransaction) -> bool:
|
2017-05-24 13:58:13 +00:00
|
|
|
|
target_min_stream_id = progress.get("target_min_stream_id_inclusive")
|
|
|
|
|
max_stream_id = progress.get("max_stream_id_exclusive")
|
|
|
|
|
|
|
|
|
|
if not target_min_stream_id or not max_stream_id:
|
|
|
|
|
txn.execute("SELECT COALESCE(MIN(stream_ordering), 0) FROM events")
|
|
|
|
|
rows = txn.fetchall()
|
|
|
|
|
target_min_stream_id = rows[0][0]
|
|
|
|
|
|
|
|
|
|
txn.execute("SELECT COALESCE(MAX(stream_ordering), 0) FROM events")
|
|
|
|
|
rows = txn.fetchall()
|
|
|
|
|
max_stream_id = rows[0][0]
|
|
|
|
|
|
|
|
|
|
min_stream_id = max_stream_id - batch_size
|
|
|
|
|
|
|
|
|
|
sql = """
|
|
|
|
|
DELETE FROM event_auth
|
|
|
|
|
WHERE event_id IN (
|
|
|
|
|
SELECT event_id FROM events
|
2021-12-02 22:42:58 +00:00
|
|
|
|
LEFT JOIN state_events AS se USING (room_id, event_id)
|
2017-05-24 13:58:13 +00:00
|
|
|
|
WHERE ? <= stream_ordering AND stream_ordering < ?
|
2021-12-02 22:42:58 +00:00
|
|
|
|
AND se.state_key IS null
|
2017-05-24 13:58:13 +00:00
|
|
|
|
)
|
|
|
|
|
"""
|
|
|
|
|
|
2019-04-03 09:07:29 +00:00
|
|
|
|
txn.execute(sql, (min_stream_id, max_stream_id))
|
2017-05-24 13:58:13 +00:00
|
|
|
|
|
|
|
|
|
new_progress = {
|
|
|
|
|
"target_min_stream_id_inclusive": target_min_stream_id,
|
|
|
|
|
"max_stream_id_exclusive": min_stream_id,
|
|
|
|
|
}
|
|
|
|
|
|
2020-08-05 20:38:57 +00:00
|
|
|
|
self.db_pool.updates._background_update_progress_txn(
|
2017-05-24 13:58:13 +00:00
|
|
|
|
txn, self.EVENT_AUTH_STATE_ONLY, new_progress
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
return min_stream_id >= target_min_stream_id
|
|
|
|
|
|
2020-08-11 21:21:13 +00:00
|
|
|
|
result = await self.db_pool.runInteraction(
|
2017-05-24 13:58:13 +00:00
|
|
|
|
self.EVENT_AUTH_STATE_ONLY, delete_event_auth
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
if not result:
|
2020-08-11 21:21:13 +00:00
|
|
|
|
await self.db_pool.updates._end_background_update(
|
2020-08-05 20:38:57 +00:00
|
|
|
|
self.EVENT_AUTH_STATE_ONLY
|
|
|
|
|
)
|
2017-05-24 13:58:13 +00:00
|
|
|
|
|
2019-07-23 13:00:55 +00:00
|
|
|
|
return batch_size
|