mirror of
https://mau.dev/maunium/synapse.git
synced 2024-08-31 06:32:11 +00:00
Merge pull request #4953 from matrix-org/rav/refactor_replication_streams
Split up replication.tcp.streams into smaller files
This commit is contained in:
commit
4eeb2c2f07
2
changelog.d/4953.misc
Normal file
2
changelog.d/4953.misc
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
Split synapse.replication.tcp.streams into smaller files.
|
||||||
|
|
@ -38,7 +38,7 @@ from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
|
|||||||
from synapse.replication.slave.storage.registration import SlavedRegistrationStore
|
from synapse.replication.slave.storage.registration import SlavedRegistrationStore
|
||||||
from synapse.replication.slave.storage.transactions import SlavedTransactionStore
|
from synapse.replication.slave.storage.transactions import SlavedTransactionStore
|
||||||
from synapse.replication.tcp.client import ReplicationClientHandler
|
from synapse.replication.tcp.client import ReplicationClientHandler
|
||||||
from synapse.replication.tcp.streams import ReceiptsStream
|
from synapse.replication.tcp.streams._base import ReceiptsStream
|
||||||
from synapse.server import HomeServer
|
from synapse.server import HomeServer
|
||||||
from synapse.storage.engines import create_engine
|
from synapse.storage.engines import create_engine
|
||||||
from synapse.types import ReadReceipt
|
from synapse.types import ReadReceipt
|
||||||
|
@ -30,7 +30,8 @@ from synapse.metrics.background_process_metrics import run_as_background_process
|
|||||||
from synapse.util.metrics import Measure, measure_func
|
from synapse.util.metrics import Measure, measure_func
|
||||||
|
|
||||||
from .protocol import ServerReplicationStreamProtocol
|
from .protocol import ServerReplicationStreamProtocol
|
||||||
from .streams import STREAMS_MAP, FederationStream
|
from .streams import STREAMS_MAP
|
||||||
|
from .streams.federation import FederationStream
|
||||||
|
|
||||||
stream_updates_counter = Counter("synapse_replication_tcp_resource_stream_updates",
|
stream_updates_counter = Counter("synapse_replication_tcp_resource_stream_updates",
|
||||||
"", ["stream_name"])
|
"", ["stream_name"])
|
||||||
|
50
synapse/replication/tcp/streams/__init__.py
Normal file
50
synapse/replication/tcp/streams/__init__.py
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright 2017 Vector Creations Ltd
|
||||||
|
# Copyright 2019 New Vector Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
"""Defines all the valid streams that clients can subscribe to, and the format
|
||||||
|
of the rows returned by each stream.
|
||||||
|
|
||||||
|
Each stream is defined by the following information:
|
||||||
|
|
||||||
|
stream name: The name of the stream
|
||||||
|
row type: The type that is used to serialise/deserialse the row
|
||||||
|
current_token: The function that returns the current token for the stream
|
||||||
|
update_function: The function that returns a list of updates between two tokens
|
||||||
|
"""
|
||||||
|
|
||||||
|
from . import _base, events, federation
|
||||||
|
|
||||||
|
STREAMS_MAP = {
|
||||||
|
stream.NAME: stream
|
||||||
|
for stream in (
|
||||||
|
events.EventsStream,
|
||||||
|
_base.BackfillStream,
|
||||||
|
_base.PresenceStream,
|
||||||
|
_base.TypingStream,
|
||||||
|
_base.ReceiptsStream,
|
||||||
|
_base.PushRulesStream,
|
||||||
|
_base.PushersStream,
|
||||||
|
_base.CachesStream,
|
||||||
|
_base.PublicRoomsStream,
|
||||||
|
_base.DeviceListsStream,
|
||||||
|
_base.ToDeviceStream,
|
||||||
|
federation.FederationStream,
|
||||||
|
_base.TagAccountDataStream,
|
||||||
|
_base.AccountDataStream,
|
||||||
|
_base.CurrentStateDeltaStream,
|
||||||
|
_base.GroupServerStream,
|
||||||
|
)
|
||||||
|
}
|
@ -1,5 +1,6 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
# Copyright 2017 Vector Creations Ltd
|
# Copyright 2017 Vector Creations Ltd
|
||||||
|
# Copyright 2019 New Vector Ltd
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
@ -13,16 +14,7 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
"""Defines all the valid streams that clients can subscribe to, and the format
|
|
||||||
of the rows returned by each stream.
|
|
||||||
|
|
||||||
Each stream is defined by the following information:
|
|
||||||
|
|
||||||
stream name: The name of the stream
|
|
||||||
row type: The type that is used to serialise/deserialse the row
|
|
||||||
current_token: The function that returns the current token for the stream
|
|
||||||
update_function: The function that returns a list of updates between two tokens
|
|
||||||
"""
|
|
||||||
import itertools
|
import itertools
|
||||||
import logging
|
import logging
|
||||||
from collections import namedtuple
|
from collections import namedtuple
|
||||||
@ -34,14 +26,6 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
MAX_EVENTS_BEHIND = 10000
|
MAX_EVENTS_BEHIND = 10000
|
||||||
|
|
||||||
|
|
||||||
EventStreamRow = namedtuple("EventStreamRow", (
|
|
||||||
"event_id", # str
|
|
||||||
"room_id", # str
|
|
||||||
"type", # str
|
|
||||||
"state_key", # str, optional
|
|
||||||
"redacts", # str, optional
|
|
||||||
))
|
|
||||||
BackfillStreamRow = namedtuple("BackfillStreamRow", (
|
BackfillStreamRow = namedtuple("BackfillStreamRow", (
|
||||||
"event_id", # str
|
"event_id", # str
|
||||||
"room_id", # str
|
"room_id", # str
|
||||||
@ -96,10 +80,6 @@ DeviceListsStreamRow = namedtuple("DeviceListsStreamRow", (
|
|||||||
ToDeviceStreamRow = namedtuple("ToDeviceStreamRow", (
|
ToDeviceStreamRow = namedtuple("ToDeviceStreamRow", (
|
||||||
"entity", # str
|
"entity", # str
|
||||||
))
|
))
|
||||||
FederationStreamRow = namedtuple("FederationStreamRow", (
|
|
||||||
"type", # str, the type of data as defined in the BaseFederationRows
|
|
||||||
"data", # dict, serialization of a federation.send_queue.BaseFederationRow
|
|
||||||
))
|
|
||||||
TagAccountDataStreamRow = namedtuple("TagAccountDataStreamRow", (
|
TagAccountDataStreamRow = namedtuple("TagAccountDataStreamRow", (
|
||||||
"user_id", # str
|
"user_id", # str
|
||||||
"room_id", # str
|
"room_id", # str
|
||||||
@ -236,20 +216,6 @@ class Stream(object):
|
|||||||
raise NotImplementedError()
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
|
||||||
class EventsStream(Stream):
|
|
||||||
"""We received a new event, or an event went from being an outlier to not
|
|
||||||
"""
|
|
||||||
NAME = "events"
|
|
||||||
ROW_TYPE = EventStreamRow
|
|
||||||
|
|
||||||
def __init__(self, hs):
|
|
||||||
store = hs.get_datastore()
|
|
||||||
self.current_token = store.get_current_events_token
|
|
||||||
self.update_function = store.get_all_new_forward_event_rows
|
|
||||||
|
|
||||||
super(EventsStream, self).__init__(hs)
|
|
||||||
|
|
||||||
|
|
||||||
class BackfillStream(Stream):
|
class BackfillStream(Stream):
|
||||||
"""We fetched some old events and either we had never seen that event before
|
"""We fetched some old events and either we had never seen that event before
|
||||||
or it went from being an outlier to not.
|
or it went from being an outlier to not.
|
||||||
@ -404,22 +370,6 @@ class ToDeviceStream(Stream):
|
|||||||
super(ToDeviceStream, self).__init__(hs)
|
super(ToDeviceStream, self).__init__(hs)
|
||||||
|
|
||||||
|
|
||||||
class FederationStream(Stream):
|
|
||||||
"""Data to be sent over federation. Only available when master has federation
|
|
||||||
sending disabled.
|
|
||||||
"""
|
|
||||||
NAME = "federation"
|
|
||||||
ROW_TYPE = FederationStreamRow
|
|
||||||
|
|
||||||
def __init__(self, hs):
|
|
||||||
federation_sender = hs.get_federation_sender()
|
|
||||||
|
|
||||||
self.current_token = federation_sender.get_current_token
|
|
||||||
self.update_function = federation_sender.get_replication_rows
|
|
||||||
|
|
||||||
super(FederationStream, self).__init__(hs)
|
|
||||||
|
|
||||||
|
|
||||||
class TagAccountDataStream(Stream):
|
class TagAccountDataStream(Stream):
|
||||||
"""Someone added/removed a tag for a room
|
"""Someone added/removed a tag for a room
|
||||||
"""
|
"""
|
||||||
@ -489,26 +439,3 @@ class GroupServerStream(Stream):
|
|||||||
self.update_function = store.get_all_groups_changes
|
self.update_function = store.get_all_groups_changes
|
||||||
|
|
||||||
super(GroupServerStream, self).__init__(hs)
|
super(GroupServerStream, self).__init__(hs)
|
||||||
|
|
||||||
|
|
||||||
STREAMS_MAP = {
|
|
||||||
stream.NAME: stream
|
|
||||||
for stream in (
|
|
||||||
EventsStream,
|
|
||||||
BackfillStream,
|
|
||||||
PresenceStream,
|
|
||||||
TypingStream,
|
|
||||||
ReceiptsStream,
|
|
||||||
PushRulesStream,
|
|
||||||
PushersStream,
|
|
||||||
CachesStream,
|
|
||||||
PublicRoomsStream,
|
|
||||||
DeviceListsStream,
|
|
||||||
ToDeviceStream,
|
|
||||||
FederationStream,
|
|
||||||
TagAccountDataStream,
|
|
||||||
AccountDataStream,
|
|
||||||
CurrentStateDeltaStream,
|
|
||||||
GroupServerStream,
|
|
||||||
)
|
|
||||||
}
|
|
40
synapse/replication/tcp/streams/events.py
Normal file
40
synapse/replication/tcp/streams/events.py
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright 2017 Vector Creations Ltd
|
||||||
|
# Copyright 2019 New Vector Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
from collections import namedtuple
|
||||||
|
|
||||||
|
from ._base import Stream
|
||||||
|
|
||||||
|
EventStreamRow = namedtuple("EventStreamRow", (
|
||||||
|
"event_id", # str
|
||||||
|
"room_id", # str
|
||||||
|
"type", # str
|
||||||
|
"state_key", # str, optional
|
||||||
|
"redacts", # str, optional
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
class EventsStream(Stream):
|
||||||
|
"""We received a new event, or an event went from being an outlier to not
|
||||||
|
"""
|
||||||
|
NAME = "events"
|
||||||
|
ROW_TYPE = EventStreamRow
|
||||||
|
|
||||||
|
def __init__(self, hs):
|
||||||
|
store = hs.get_datastore()
|
||||||
|
self.current_token = store.get_current_events_token
|
||||||
|
self.update_function = store.get_all_new_forward_event_rows
|
||||||
|
|
||||||
|
super(EventsStream, self).__init__(hs)
|
39
synapse/replication/tcp/streams/federation.py
Normal file
39
synapse/replication/tcp/streams/federation.py
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright 2017 Vector Creations Ltd
|
||||||
|
# Copyright 2019 New Vector Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
from collections import namedtuple
|
||||||
|
|
||||||
|
from ._base import Stream
|
||||||
|
|
||||||
|
FederationStreamRow = namedtuple("FederationStreamRow", (
|
||||||
|
"type", # str, the type of data as defined in the BaseFederationRows
|
||||||
|
"data", # dict, serialization of a federation.send_queue.BaseFederationRow
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
class FederationStream(Stream):
|
||||||
|
"""Data to be sent over federation. Only available when master has federation
|
||||||
|
sending disabled.
|
||||||
|
"""
|
||||||
|
NAME = "federation"
|
||||||
|
ROW_TYPE = FederationStreamRow
|
||||||
|
|
||||||
|
def __init__(self, hs):
|
||||||
|
federation_sender = hs.get_federation_sender()
|
||||||
|
|
||||||
|
self.current_token = federation_sender.get_current_token
|
||||||
|
self.update_function = federation_sender.get_replication_rows
|
||||||
|
|
||||||
|
super(FederationStream, self).__init__(hs)
|
@ -12,7 +12,7 @@
|
|||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
from synapse.replication.tcp.streams import ReceiptsStreamRow
|
from synapse.replication.tcp.streams._base import ReceiptsStreamRow
|
||||||
|
|
||||||
from tests.replication.tcp.streams._base import BaseStreamTestCase
|
from tests.replication.tcp.streams._base import BaseStreamTestCase
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user