Matrix-Mirrors/forked-synapse
1
0
Fork 0
mirror of https://mau.dev/maunium/synapse.git synced 2024-10-01 01:36:05 -04:00
Code Issues Packages Projects Releases Wiki Activity

Linearize fetching of gaps on incoming events

Browse Source 
This potentially stops the server from doing multiple requests for the
same data.
This commit is contained in:
Erik Johnston 2016-06-15 15:12:59 +01:00
parent 15bf3e3376
commit d41a1a91d3
4 changed files with 52 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
synapse/federation/federation_base.py
View File

@ -31,6 +31,9 @@ logger = logging.getLogger(__name__)
class FederationBase(object): class FederationBase(object):
def __init__(self, hs):
pass
@defer.inlineCallbacks @defer.inlineCallbacks
def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False, def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False,
include_none=False): include_none=False):

2
synapse/federation/federation_client.py
View File

@ -52,6 +52,8 @@ sent_queries_counter = metrics.register_counter("sent_queries", labels=["type"])
class FederationClient(FederationBase): class FederationClient(FederationBase):
def __init__(self, hs):
super(FederationClient, self).__init__(hs)
def start_get_pdu_cache(self): def start_get_pdu_cache(self):
self._get_pdu_cache = ExpiringCache( self._get_pdu_cache = ExpiringCache(

76
synapse/federation/federation_server.py
View File

@ -19,6 +19,7 @@ from twisted.internet import defer
from .federation_base import FederationBase from .federation_base import FederationBase
from .units import Transaction, Edu from .units import Transaction, Edu
from synapse.util.async import Linearizer
from synapse.util.logutils import log_function from synapse.util.logutils import log_function
from synapse.events import FrozenEvent from synapse.events import FrozenEvent
import synapse.metrics import synapse.metrics
@ -44,6 +45,11 @@ received_queries_counter = metrics.register_counter("received_queries", labels=[
class FederationServer(FederationBase): class FederationServer(FederationBase):
def __init__(self, hs):
super(FederationServer, self).__init__(hs)
self._room_pdu_linearizer = Linearizer()
def set_handler(self, handler): def set_handler(self, handler):
"""Sets the handler that the replication layer will use to communicate """Sets the handler that the replication layer will use to communicate
receipt of new PDUs from other home servers. The required methods are receipt of new PDUs from other home servers. The required methods are
@ -491,43 +497,51 @@ class FederationServer(FederationBase):
pdu.internal_metadata.outlier = True pdu.internal_metadata.outlier = True
elif min_depth and pdu.depth > min_depth: elif min_depth and pdu.depth > min_depth:
if get_missing and prevs - seen: if get_missing and prevs - seen:
latest = yield self.store.get_latest_event_ids_in_room( # If we're missing stuff, ensure we only fetch stuff one
pdu.room_id # at a time.
) with (yield self._room_pdu_linearizer.queue(pdu.room_id)):
# We recalculate seen, since it may have changed.
have_seen = yield self.store.have_events(prevs)
seen = set(have_seen.keys())
# We add the prev events that we have seen to the latest if prevs - seen:
# list to ensure the remote server doesn't give them to us latest = yield self.store.get_latest_event_ids_in_room(
latest = set(latest) pdu.room_id
latest |= seen )
logger.info( # We add the prev events that we have seen to the latest
"Missing %d events for room %r: %r...", # list to ensure the remote server doesn't give them to us
len(prevs - seen), pdu.room_id, list(prevs - seen)[:5] latest = set(latest)
) latest |= seen
missing_events = yield self.get_missing_events( logger.info(
origin, "Missing %d events for room %r: %r...",
pdu.room_id, len(prevs - seen), pdu.room_id, list(prevs - seen)[:5]
earliest_events_ids=list(latest), )
latest_events=[pdu],
limit=10,
min_depth=min_depth,
)
# We want to sort these by depth so we process them and missing_events = yield self.get_missing_events(
# tell clients about them in order. origin,
missing_events.sort(key=lambda x: x.depth) pdu.room_id,
earliest_events_ids=list(latest),
latest_events=[pdu],
limit=10,
min_depth=min_depth,
)
for e in missing_events: # We want to sort these by depth so we process them and
yield self._handle_new_pdu( # tell clients about them in order.
origin, missing_events.sort(key=lambda x: x.depth)
e,
get_missing=False
)
have_seen = yield self.store.have_events( for e in missing_events:
[ev for ev, _ in pdu.prev_events] yield self._handle_new_pdu(
) origin,
e,
get_missing=False
)
have_seen = yield self.store.have_events(
[ev for ev, _ in pdu.prev_events]
)
prevs = {e_id for e_id, _ in pdu.prev_events} prevs = {e_id for e_id, _ in pdu.prev_events}
seen = set(have_seen.keys()) seen = set(have_seen.keys())

2
synapse/federation/replication.py
View File

@ -72,5 +72,7 @@ class ReplicationLayer(FederationClient, FederationServer):
self.hs = hs self.hs = hs
super(ReplicationLayer, self).__init__(hs)
def __str__(self): def __str__(self):
return "<ReplicationLayer(%s)>" % self.server_name return "<ReplicationLayer(%s)>" % self.server_name