mirror of
https://git.anonymousland.org/anonymousland/synapse.git
synced 2024-09-07 02:31:59 +00:00
Linearize fetching of gaps on incoming events
This potentially stops the server from doing multiple requests for the same data.
This commit is contained in:
parent
15bf3e3376
commit
d41a1a91d3
@ -31,6 +31,9 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
|
|
||||||
class FederationBase(object):
|
class FederationBase(object):
|
||||||
|
def __init__(self, hs):
|
||||||
|
pass
|
||||||
|
|
||||||
@defer.inlineCallbacks
|
@defer.inlineCallbacks
|
||||||
def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False,
|
def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False,
|
||||||
include_none=False):
|
include_none=False):
|
||||||
|
@ -52,6 +52,8 @@ sent_queries_counter = metrics.register_counter("sent_queries", labels=["type"])
|
|||||||
|
|
||||||
|
|
||||||
class FederationClient(FederationBase):
|
class FederationClient(FederationBase):
|
||||||
|
def __init__(self, hs):
|
||||||
|
super(FederationClient, self).__init__(hs)
|
||||||
|
|
||||||
def start_get_pdu_cache(self):
|
def start_get_pdu_cache(self):
|
||||||
self._get_pdu_cache = ExpiringCache(
|
self._get_pdu_cache = ExpiringCache(
|
||||||
|
@ -19,6 +19,7 @@ from twisted.internet import defer
|
|||||||
from .federation_base import FederationBase
|
from .federation_base import FederationBase
|
||||||
from .units import Transaction, Edu
|
from .units import Transaction, Edu
|
||||||
|
|
||||||
|
from synapse.util.async import Linearizer
|
||||||
from synapse.util.logutils import log_function
|
from synapse.util.logutils import log_function
|
||||||
from synapse.events import FrozenEvent
|
from synapse.events import FrozenEvent
|
||||||
import synapse.metrics
|
import synapse.metrics
|
||||||
@ -44,6 +45,11 @@ received_queries_counter = metrics.register_counter("received_queries", labels=[
|
|||||||
|
|
||||||
|
|
||||||
class FederationServer(FederationBase):
|
class FederationServer(FederationBase):
|
||||||
|
def __init__(self, hs):
|
||||||
|
super(FederationServer, self).__init__(hs)
|
||||||
|
|
||||||
|
self._room_pdu_linearizer = Linearizer()
|
||||||
|
|
||||||
def set_handler(self, handler):
|
def set_handler(self, handler):
|
||||||
"""Sets the handler that the replication layer will use to communicate
|
"""Sets the handler that the replication layer will use to communicate
|
||||||
receipt of new PDUs from other home servers. The required methods are
|
receipt of new PDUs from other home servers. The required methods are
|
||||||
@ -491,43 +497,51 @@ class FederationServer(FederationBase):
|
|||||||
pdu.internal_metadata.outlier = True
|
pdu.internal_metadata.outlier = True
|
||||||
elif min_depth and pdu.depth > min_depth:
|
elif min_depth and pdu.depth > min_depth:
|
||||||
if get_missing and prevs - seen:
|
if get_missing and prevs - seen:
|
||||||
latest = yield self.store.get_latest_event_ids_in_room(
|
# If we're missing stuff, ensure we only fetch stuff one
|
||||||
pdu.room_id
|
# at a time.
|
||||||
)
|
with (yield self._room_pdu_linearizer.queue(pdu.room_id)):
|
||||||
|
# We recalculate seen, since it may have changed.
|
||||||
|
have_seen = yield self.store.have_events(prevs)
|
||||||
|
seen = set(have_seen.keys())
|
||||||
|
|
||||||
# We add the prev events that we have seen to the latest
|
if prevs - seen:
|
||||||
# list to ensure the remote server doesn't give them to us
|
latest = yield self.store.get_latest_event_ids_in_room(
|
||||||
latest = set(latest)
|
pdu.room_id
|
||||||
latest |= seen
|
)
|
||||||
|
|
||||||
logger.info(
|
# We add the prev events that we have seen to the latest
|
||||||
"Missing %d events for room %r: %r...",
|
# list to ensure the remote server doesn't give them to us
|
||||||
len(prevs - seen), pdu.room_id, list(prevs - seen)[:5]
|
latest = set(latest)
|
||||||
)
|
latest |= seen
|
||||||
|
|
||||||
missing_events = yield self.get_missing_events(
|
logger.info(
|
||||||
origin,
|
"Missing %d events for room %r: %r...",
|
||||||
pdu.room_id,
|
len(prevs - seen), pdu.room_id, list(prevs - seen)[:5]
|
||||||
earliest_events_ids=list(latest),
|
)
|
||||||
latest_events=[pdu],
|
|
||||||
limit=10,
|
|
||||||
min_depth=min_depth,
|
|
||||||
)
|
|
||||||
|
|
||||||
# We want to sort these by depth so we process them and
|
missing_events = yield self.get_missing_events(
|
||||||
# tell clients about them in order.
|
origin,
|
||||||
missing_events.sort(key=lambda x: x.depth)
|
pdu.room_id,
|
||||||
|
earliest_events_ids=list(latest),
|
||||||
|
latest_events=[pdu],
|
||||||
|
limit=10,
|
||||||
|
min_depth=min_depth,
|
||||||
|
)
|
||||||
|
|
||||||
for e in missing_events:
|
# We want to sort these by depth so we process them and
|
||||||
yield self._handle_new_pdu(
|
# tell clients about them in order.
|
||||||
origin,
|
missing_events.sort(key=lambda x: x.depth)
|
||||||
e,
|
|
||||||
get_missing=False
|
|
||||||
)
|
|
||||||
|
|
||||||
have_seen = yield self.store.have_events(
|
for e in missing_events:
|
||||||
[ev for ev, _ in pdu.prev_events]
|
yield self._handle_new_pdu(
|
||||||
)
|
origin,
|
||||||
|
e,
|
||||||
|
get_missing=False
|
||||||
|
)
|
||||||
|
|
||||||
|
have_seen = yield self.store.have_events(
|
||||||
|
[ev for ev, _ in pdu.prev_events]
|
||||||
|
)
|
||||||
|
|
||||||
prevs = {e_id for e_id, _ in pdu.prev_events}
|
prevs = {e_id for e_id, _ in pdu.prev_events}
|
||||||
seen = set(have_seen.keys())
|
seen = set(have_seen.keys())
|
||||||
|
@ -72,5 +72,7 @@ class ReplicationLayer(FederationClient, FederationServer):
|
|||||||
|
|
||||||
self.hs = hs
|
self.hs = hs
|
||||||
|
|
||||||
|
super(ReplicationLayer, self).__init__(hs)
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return "<ReplicationLayer(%s)>" % self.server_name
|
return "<ReplicationLayer(%s)>" % self.server_name
|
||||||
|
Loading…
Reference in New Issue
Block a user