mirror of
https://git.anonymousland.org/anonymousland/synapse.git
synced 2025-05-02 11:26:09 -04:00
Merge pull request #305 from matrix-org/markjh/v2_sync_api
Update the v2 sync API to work as specified in the current spec.
This commit is contained in:
Mark Haines
commit
a059760954
3 changed files with 212 additions and 171 deletions
|
|
@ -28,21 +28,28 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
SyncConfig = collections.namedtuple("SyncConfig", [
|
||||
"user",
|
||||
"limit",
|
||||
"gap",
|
||||
"sort",
|
||||
"backfill",
|
||||
"filter",
|
||||
])
|
||||
|
||||
|
||||
class RoomSyncResult(collections.namedtuple("RoomSyncResult", [
|
||||
"room_id",
|
||||
"limited",
|
||||
"published",
|
||||
"events",
|
||||
"state",
|
||||
class TimelineBatch(collections.namedtuple("TimelineBatch", [
|
||||
"prev_batch",
|
||||
"events",
|
||||
"limited",
|
||||
])):
|
||||
__slots__ = []
|
||||
|
||||
def __nonzero__(self):
|
||||
"""Make the result appear empty if there are no updates. This is used
|
||||
to tell if room needs to be part of the sync result.
|
||||
"""
|
||||
return bool(self.events)
|
||||
|
||||
|
||||
class JoinedSyncResult(collections.namedtuple("JoinedSyncResult", [
|
||||
"room_id",
|
||||
"timeline",
|
||||
"state",
|
||||
"ephemeral",
|
||||
])):
|
||||
__slots__ = []
|
||||
|
|
@ -51,14 +58,21 @@ class RoomSyncResult(collections.namedtuple("RoomSyncResult", [
|
|||
"""Make the result appear empty if there are no updates. This is used
|
||||
to tell if room needs to be part of the sync result.
|
||||
"""
|
||||
return bool(self.events or self.state or self.ephemeral)
|
||||
return bool(self.timeline or self.state or self.ephemeral)
|
||||
|
||||
|
||||
class InvitedSyncResult(collections.namedtuple("InvitedSyncResult", [
|
||||
"room_id",
|
||||
"invite",
|
||||
])):
|
||||
__slots__ = []
|
||||
|
||||
|
||||
class SyncResult(collections.namedtuple("SyncResult", [
|
||||
"next_batch", # Token for the next sync
|
||||
"private_user_data", # List of private events for the user.
|
||||
"public_user_data", # List of public events for all users.
|
||||
"rooms", # RoomSyncResult for each room.
|
||||
"presence", # List of presence events for the user.
|
||||
"joined", # JoinedSyncResult for each joined room.
|
||||
"invited", # InvitedSyncResult for each invited room.
|
||||
])):
|
||||
__slots__ = []
|
||||
|
||||
|
|
@ -68,7 +82,7 @@ class SyncResult(collections.namedtuple("SyncResult", [
|
|||
events.
|
||||
"""
|
||||
return bool(
|
||||
self.private_user_data or self.public_user_data or self.rooms
|
||||
self.presence or self.joined or self.invited
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -108,8 +122,8 @@ class SyncHandler(BaseHandler):
|
|||
)
|
||||
|
||||
result = yield self.notifier.wait_for_events(
|
||||
sync_config.user, room_ids,
|
||||
sync_config.filter, timeout, current_sync_callback
|
||||
sync_config.user, room_ids, timeout, current_sync_callback,
|
||||
from_token=since_token
|
||||
)
|
||||
defer.returnValue(result)
|
||||
|
||||
|
|
@ -121,11 +135,7 @@ class SyncHandler(BaseHandler):
|
|||
if since_token is None:
|
||||
return self.initial_sync(sync_config)
|
||||
else:
|
||||
if sync_config.gap:
|
||||
return self.incremental_sync_with_gap(sync_config, since_token)
|
||||
else:
|
||||
# TODO(mjark): Handle gapless sync
|
||||
raise NotImplementedError()
|
||||
return self.incremental_sync_with_gap(sync_config, since_token)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def initial_sync(self, sync_config):
|
||||
|
|
@ -133,12 +143,6 @@ class SyncHandler(BaseHandler):
|
|||
Returns:
|
||||
A Deferred SyncResult.
|
||||
"""
|
||||
if sync_config.sort == "timeline,desc":
|
||||
# TODO(mjark): Handle going through events in reverse order?.
|
||||
# What does "most recent events" mean when applying the limits mean
|
||||
# in this case?
|
||||
raise NotImplementedError()
|
||||
|
||||
now_token = yield self.event_sources.get_current_token()
|
||||
|
||||
presence_stream = self.event_sources.sources["presence"]
|
||||
|
|
@ -155,33 +159,36 @@ class SyncHandler(BaseHandler):
|
|||
membership_list=[Membership.INVITE, Membership.JOIN]
|
||||
)
|
||||
|
||||
# TODO (mjark): Does public mean "published"?
|
||||
published_rooms = yield self.store.get_rooms(is_public=True)
|
||||
published_room_ids = set(r["room_id"] for r in published_rooms)
|
||||
|
||||
rooms = []
|
||||
joined = []
|
||||
invited = []
|
||||
for event in room_list:
|
||||
room_sync = yield self.initial_sync_for_room(
|
||||
event.room_id, sync_config, now_token, published_room_ids
|
||||
)
|
||||
rooms.append(room_sync)
|
||||
if event.membership == Membership.JOIN:
|
||||
room_sync = yield self.initial_sync_for_joined_room(
|
||||
event.room_id, sync_config, now_token,
|
||||
)
|
||||
joined.append(room_sync)
|
||||
elif event.membership == Membership.INVITE:
|
||||
invite = yield self.store.get_event(event.event_id)
|
||||
invited.append(InvitedSyncResult(
|
||||
room_id=event.room_id,
|
||||
invite=invite,
|
||||
))
|
||||
|
||||
defer.returnValue(SyncResult(
|
||||
public_user_data=presence,
|
||||
private_user_data=[],
|
||||
rooms=rooms,
|
||||
presence=presence,
|
||||
joined=joined,
|
||||
invited=invited,
|
||||
next_batch=now_token,
|
||||
))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def initial_sync_for_room(self, room_id, sync_config, now_token,
|
||||
published_room_ids):
|
||||
def initial_sync_for_joined_room(self, room_id, sync_config, now_token):
|
||||
"""Sync a room for a client which is starting without any state
|
||||
Returns:
|
||||
A Deferred RoomSyncResult.
|
||||
A Deferred JoinedSyncResult.
|
||||
"""
|
||||
|
||||
recents, prev_batch_token, limited = yield self.load_filtered_recents(
|
||||
batch = yield self.load_filtered_recents(
|
||||
room_id, sync_config, now_token,
|
||||
)
|
||||
|
||||
|
|
@ -190,13 +197,10 @@ class SyncHandler(BaseHandler):
|
|||
)
|
||||
current_state_events = current_state.values()
|
||||
|
||||
defer.returnValue(RoomSyncResult(
|
||||
defer.returnValue(JoinedSyncResult(
|
||||
room_id=room_id,
|
||||
published=room_id in published_room_ids,
|
||||
events=recents,
|
||||
prev_batch=prev_batch_token,
|
||||
timeline=batch,
|
||||
state=current_state_events,
|
||||
limited=limited,
|
||||
ephemeral=[],
|
||||
))
|
||||
|
||||
|
|
@ -207,19 +211,13 @@ class SyncHandler(BaseHandler):
|
|||
Returns:
|
||||
A Deferred SyncResult.
|
||||
"""
|
||||
if sync_config.sort == "timeline,desc":
|
||||
# TODO(mjark): Handle going through events in reverse order?.
|
||||
# What does "most recent events" mean when applying the limits mean
|
||||
# in this case?
|
||||
raise NotImplementedError()
|
||||
|
||||
now_token = yield self.event_sources.get_current_token()
|
||||
|
||||
presence_source = self.event_sources.sources["presence"]
|
||||
presence, presence_key = yield presence_source.get_new_events_for_user(
|
||||
user=sync_config.user,
|
||||
from_key=since_token.presence_key,
|
||||
limit=sync_config.limit,
|
||||
limit=sync_config.filter.presence_limit(),
|
||||
)
|
||||
now_token = now_token.copy_and_replace("presence_key", presence_key)
|
||||
|
||||
|
|
@ -227,7 +225,7 @@ class SyncHandler(BaseHandler):
|
|||
typing, typing_key = yield typing_source.get_new_events_for_user(
|
||||
user=sync_config.user,
|
||||
from_key=since_token.typing_key,
|
||||
limit=sync_config.limit,
|
||||
limit=sync_config.filter.ephemeral_limit(),
|
||||
)
|
||||
now_token = now_token.copy_and_replace("typing_key", typing_key)
|
||||
|
||||
|
|
@ -242,33 +240,37 @@ class SyncHandler(BaseHandler):
|
|||
)
|
||||
if app_service:
|
||||
rooms = yield self.store.get_app_service_rooms(app_service)
|
||||
room_ids = set(r.room_id for r in rooms)
|
||||
joined_room_ids = set(r.room_id for r in rooms)
|
||||
else:
|
||||
room_ids = yield rm_handler.get_joined_rooms_for_user(
|
||||
joined_room_ids = yield rm_handler.get_joined_rooms_for_user(
|
||||
sync_config.user
|
||||
)
|
||||
|
||||
# TODO (mjark): Does public mean "published"?
|
||||
published_rooms = yield self.store.get_rooms(is_public=True)
|
||||
published_room_ids = set(r["room_id"] for r in published_rooms)
|
||||
timeline_limit = sync_config.filter.timeline_limit()
|
||||
|
||||
room_events, _ = yield self.store.get_room_events_stream(
|
||||
sync_config.user.to_string(),
|
||||
from_key=since_token.room_key,
|
||||
to_key=now_token.room_key,
|
||||
room_id=None,
|
||||
limit=sync_config.limit + 1,
|
||||
limit=timeline_limit + 1,
|
||||
)
|
||||
|
||||
rooms = []
|
||||
if len(room_events) <= sync_config.limit:
|
||||
joined = []
|
||||
if len(room_events) <= timeline_limit:
|
||||
# There is no gap in any of the rooms. Therefore we can just
|
||||
# partition the new events by room and return them.
|
||||
invite_events = []
|
||||
events_by_room_id = {}
|
||||
for event in room_events:
|
||||
events_by_room_id.setdefault(event.room_id, []).append(event)
|
||||
if event.room_id not in joined_room_ids:
|
||||
if (event.type == EventTypes.Member
|
||||
and event.membership == Membership.INVITE
|
||||
and event.state_key == sync_config.user.to_string()):
|
||||
invite_events.append(event)
|
||||
|
||||
for room_id in room_ids:
|
||||
for room_id in joined_room_ids:
|
||||
recents = events_by_room_id.get(room_id, [])
|
||||
state = [event for event in recents if event.is_state()]
|
||||
if recents:
|
||||
|
|
@ -282,30 +284,40 @@ class SyncHandler(BaseHandler):
|
|||
sync_config, room_id, state
|
||||
)
|
||||
|
||||
room_sync = RoomSyncResult(
|
||||
room_sync = JoinedSyncResult(
|
||||
room_id=room_id,
|
||||
published=room_id in published_room_ids,
|
||||
events=recents,
|
||||
prev_batch=prev_batch,
|
||||
timeline=TimelineBatch(
|
||||
events=recents,
|
||||
prev_batch=prev_batch,
|
||||
limited=False,
|
||||
),
|
||||
state=state,
|
||||
limited=False,
|
||||
ephemeral=typing_by_room.get(room_id, [])
|
||||
)
|
||||
if room_sync:
|
||||
rooms.append(room_sync)
|
||||
joined.append(room_sync)
|
||||
else:
|
||||
for room_id in room_ids:
|
||||
invite_events = yield self.store.get_invites_for_user(
|
||||
sync_config.user.to_string()
|
||||
)
|
||||
|
||||
for room_id in joined_room_ids:
|
||||
room_sync = yield self.incremental_sync_with_gap_for_room(
|
||||
room_id, sync_config, since_token, now_token,
|
||||
published_room_ids, typing_by_room
|
||||
typing_by_room
|
||||
)
|
||||
if room_sync:
|
||||
rooms.append(room_sync)
|
||||
joined.append(room_sync)
|
||||
|
||||
invited = [
|
||||
InvitedSyncResult(room_id=event.room_id, invite=event)
|
||||
for event in invite_events
|
||||
]
|
||||
|
||||
defer.returnValue(SyncResult(
|
||||
public_user_data=presence,
|
||||
private_user_data=[],
|
||||
rooms=rooms,
|
||||
presence=presence,
|
||||
joined=joined,
|
||||
invited=invited,
|
||||
next_batch=now_token,
|
||||
))
|
||||
|
||||
|
|
@ -361,12 +373,13 @@ class SyncHandler(BaseHandler):
|
|||
limited = True
|
||||
recents = []
|
||||
filtering_factor = 2
|
||||
load_limit = max(sync_config.limit * filtering_factor, 100)
|
||||
timeline_limit = sync_config.filter.timeline_limit()
|
||||
load_limit = max(timeline_limit * filtering_factor, 100)
|
||||
max_repeat = 3 # Only try a few times per room, otherwise
|
||||
room_key = now_token.room_key
|
||||
end_key = room_key
|
||||
|
||||
while limited and len(recents) < sync_config.limit and max_repeat:
|
||||
while limited and len(recents) < timeline_limit and max_repeat:
|
||||
events, keys = yield self.store.get_recent_events_for_room(
|
||||
room_id,
|
||||
limit=load_limit + 1,
|
||||
|
|
@ -375,7 +388,7 @@ class SyncHandler(BaseHandler):
|
|||
)
|
||||
(room_key, _) = keys
|
||||
end_key = "s" + room_key.split('-')[-1]
|
||||
loaded_recents = sync_config.filter.filter_room_events(events)
|
||||
loaded_recents = sync_config.filter.filter_room_timeline(events)
|
||||
loaded_recents = yield self._filter_events_for_client(
|
||||
sync_config.user.to_string(), room_id, loaded_recents,
|
||||
)
|
||||
|
|
@ -385,34 +398,37 @@ class SyncHandler(BaseHandler):
|
|||
limited = False
|
||||
max_repeat -= 1
|
||||
|
||||
if len(recents) > sync_config.limit:
|
||||
recents = recents[-sync_config.limit:]
|
||||
if len(recents) > timeline_limit:
|
||||
limited = True
|
||||
recents = recents[-timeline_limit:]
|
||||
room_key = recents[0].internal_metadata.before
|
||||
|
||||
prev_batch_token = now_token.copy_and_replace(
|
||||
"room_key", room_key
|
||||
)
|
||||
|
||||
defer.returnValue((recents, prev_batch_token, limited))
|
||||
defer.returnValue(TimelineBatch(
|
||||
events=recents, prev_batch=prev_batch_token, limited=limited
|
||||
))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def incremental_sync_with_gap_for_room(self, room_id, sync_config,
|
||||
since_token, now_token,
|
||||
published_room_ids, typing_by_room):
|
||||
typing_by_room):
|
||||
""" Get the incremental delta needed to bring the client up to date for
|
||||
the room. Gives the client the most recent events and the changes to
|
||||
state.
|
||||
Returns:
|
||||
A Deferred RoomSyncResult
|
||||
A Deferred JoinedSyncResult
|
||||
"""
|
||||
|
||||
# TODO(mjark): Check for redactions we might have missed.
|
||||
|
||||
recents, prev_batch_token, limited = yield self.load_filtered_recents(
|
||||
batch = yield self.load_filtered_recents(
|
||||
room_id, sync_config, now_token, since_token,
|
||||
)
|
||||
|
||||
logging.debug("Recents %r", recents)
|
||||
logging.debug("Recents %r", batch)
|
||||
|
||||
# TODO(mjark): This seems racy since this isn't being passed a
|
||||
# token to indicate what point in the stream this is
|
||||
|
|
@ -435,13 +451,10 @@ class SyncHandler(BaseHandler):
|
|||
sync_config, room_id, state_events_delta
|
||||
)
|
||||
|
||||
room_sync = RoomSyncResult(
|
||||
room_sync = JoinedSyncResult(
|
||||
room_id=room_id,
|
||||
published=room_id in published_room_ids,
|
||||
events=recents,
|
||||
prev_batch=prev_batch_token,
|
||||
timeline=batch,
|
||||
state=state_events_delta,
|
||||
limited=limited,
|
||||
ephemeral=typing_by_room.get(room_id, [])
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue