From 50c8e3fcda0a892e9205be046bf8e0a931514e15 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 10 Nov 2014 18:07:55 +0000 Subject: [PATCH 01/10] Initial (empty) test that room initialSync at least returns 200 OK --- tests/rest/test_rooms.py | 53 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 1ce9b8a83..69dade454 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -940,6 +940,59 @@ class RoomMessagesTestCase(RestTestCase): (code, response) = yield self.mock_resource.trigger("PUT", path, content) self.assertEquals(200, code, msg=str(response)) + +class RoomInitialSyncTestCase(RestTestCase): + """ Tests /rooms/$room_id/initialSync. """ + user_id = "@sid1:red" + + @defer.inlineCallbacks + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + self.auth_user_id = self.user_id + + state_handler = Mock(spec=["handle_new_event"]) + state_handler.handle_new_event.return_value = True + + persistence_service = Mock(spec=["get_latest_pdus_in_context"]) + persistence_service.get_latest_pdus_in_context.return_value = [] + + hs = HomeServer( + "red", + db_pool=None, + http_client=None, + datastore=MemoryDataStore(), + replication_layer=Mock(), + state_handler=state_handler, + persistence_service=persistence_service, + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=NonCallableMock(), + ) + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + + hs.get_handlers().federation_handler = Mock() + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(self.auth_user_id), + "admin": False, + "device_id": None, + } + hs.get_auth().get_user_by_token = _get_user_by_token + + synapse.rest.room.register_servlets(hs, self.mock_resource) + + # create the room + self.room_id = yield self.create_room_as(self.user_id) + + @defer.inlineCallbacks + def test_initial_sync(self): + (code, response) = yield self.mock_resource.trigger_get( + "/rooms/%s/initialSync" % self.room_id) + self.assertEquals(200, code) + # (code, response) = yield self.mock_resource.trigger("GET", path, None) # self.assertEquals(200, code, msg=str(response)) # self.assert_dict(json.loads(content), response) From 269f80bf8ecc4a89c2bb71f0872893bb53a252ca Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 10 Nov 2014 19:02:19 +0000 Subject: [PATCH 02/10] Have room initialSync return the room's room_id --- synapse/handlers/message.py | 11 +++++++++++ synapse/rest/room.py | 27 ++++++--------------------- tests/rest/test_rooms.py | 2 ++ 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 72894869e..bb1290fd0 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -331,4 +331,15 @@ class MessageHandler(BaseHandler): defer.returnValue(ret) + @defer.inlineCallbacks + def snapshot_room(self, user_id, room_id, pagin_config=None, + feedback=False): + yield self.auth.check_joined_room(room_id, user_id) + defer.returnValue({ + #"membership": membership, + "room_id": room_id, + #"messages": messages, + #"state": state, + #"presence": presence, + }) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 772496706..b762de1cb 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -353,27 +353,12 @@ class RoomInitialSyncRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - yield self.auth.get_user_by_req(request) - # TODO: Get all the initial sync data for this room and return in the - # same format as initial sync, that is: - # { - # membership: join, - # messages: [ - # chunk: [ msg events ], - # start: s_tok, - # end: e_tok - # ], - # room_id: foo, - # state: [ - # { state event } , { state event } - # ] - # } - # Probably worth keeping the keys room_id and membership for parity - # with /initialSync even though they must be joined to sync this and - # know the room ID, so clients can reuse the same code (room_id and - # membership are MANDATORY for /initialSync, so the code will expect - # it to be there) - defer.returnValue((200, {})) + user = yield self.auth.get_user_by_req(request) + events = yield self.handlers.message_handler.snapshot_room( + room_id=room_id, + user_id=user.to_string(), + ) + defer.returnValue((200, events)) class RoomTriggerBackfill(RestServlet): diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 69dade454..129f4d49f 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -993,6 +993,8 @@ class RoomInitialSyncTestCase(RestTestCase): "/rooms/%s/initialSync" % self.room_id) self.assertEquals(200, code) + self.assertEquals(self.room_id, response["room_id"]) + # (code, response) = yield self.mock_resource.trigger("GET", path, None) # self.assertEquals(200, code, msg=str(response)) # self.assert_dict(json.loads(content), response) From 1fd81391385703475bd5c30e7617228cf33c7087 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 10 Nov 2014 19:29:58 +0000 Subject: [PATCH 03/10] Put room state in room initialSync output - I guess this is right; I really can't find any other tests similar... --- synapse/handlers/message.py | 7 +++++-- tests/rest/test_rooms.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index bb1290fd0..d8764a893 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -336,10 +336,13 @@ class MessageHandler(BaseHandler): feedback=False): yield self.auth.check_joined_room(room_id, user_id) + state_tuples = yield self.store.get_current_state(room_id) + state = [self.hs.serialize_event(x) for x in state_tuples] + defer.returnValue({ #"membership": membership, "room_id": room_id, #"messages": messages, - #"state": state, - #"presence": presence, + "state": state, + #"presence": presence }) diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 129f4d49f..1e8b16c36 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -984,6 +984,29 @@ class RoomInitialSyncTestCase(RestTestCase): synapse.rest.room.register_servlets(hs, self.mock_resource) + # MemoryDataStore's get_current_state() isn't even approximately + # right. Longterm we'll be killing it in favour of a real + # SQLite :memory:-based test. For now lets patch it + efact = hs.get_event_factory() + def get_current_state(room_id): + """ + TODO: these probably need content + ('!WxMBxqtWbMMdMEkRqY:red', 'm.room.send_event_level', ''), + ('!WxMBxqtWbMMdMEkRqY:red', 'm.room.power_levels', ''), + ('!WxMBxqtWbMMdMEkRqY:red', 'm.room.ops_levels', ''), + ('!WxMBxqtWbMMdMEkRqY:red', 'm.room.member', '@sid1:red'), + ('!WxMBxqtWbMMdMEkRqY:red', 'm.room.add_state_level', '')] + """ + return [ + efact.create_event(etype="m.room.create", room_id=room_id, + content={}, + ), + efact.create_event(etype="m.room.join_rules", room_id=room_id, + content={}, + ), + ] + hs.datastore.get_current_state = get_current_state + # create the room self.room_id = yield self.create_room_as(self.user_id) @@ -995,6 +1018,18 @@ class RoomInitialSyncTestCase(RestTestCase): self.assertEquals(self.room_id, response["room_id"]) + # Room state is easier to assert on if we unpack it into a dict + state = {} + for event in response["state"]: + if "state_key" not in event: + continue + t = event["type"] + if t not in state: + state[t] = [] + state[t].append(event) + + self.assertTrue("m.room.create" in state) + # (code, response) = yield self.mock_resource.trigger("GET", path, None) # self.assertEquals(200, code, msg=str(response)) # self.assert_dict(json.loads(content), response) From c23afed39a3a2796a53caaec19de2d53873956e5 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 10 Nov 2014 19:34:47 +0000 Subject: [PATCH 04/10] Include room membership in room initialSync --- synapse/handlers/message.py | 8 +++++++- tests/rest/test_rooms.py | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index d8764a893..811e280c2 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -336,11 +336,17 @@ class MessageHandler(BaseHandler): feedback=False): yield self.auth.check_joined_room(room_id, user_id) + # TODO: These concurrently state_tuples = yield self.store.get_current_state(room_id) state = [self.hs.serialize_event(x) for x in state_tuples] + member_event = (yield self.store.get_room_member( + user_id=user_id, + room_id=room_id + )) + defer.returnValue({ - #"membership": membership, + "membership": member_event.membership, "room_id": room_id, #"messages": messages, "state": state, diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 1e8b16c36..29f5776f4 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -1017,6 +1017,7 @@ class RoomInitialSyncTestCase(RestTestCase): self.assertEquals(200, code) self.assertEquals(self.room_id, response["room_id"]) + self.assertEquals("join", response["membership"]) # Room state is easier to assert on if we unpack it into a dict state = {} From 17f977a9de458e375a8e09f8f68ec4393e0dd1bb Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 18 Nov 2014 14:07:51 +0000 Subject: [PATCH 05/10] Include 'messages' snapshot in room initialSync --- synapse/handlers/message.py | 21 ++++++++++++++++++++- tests/rest/test_rooms.py | 4 ++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index b77d9d164..778cdb231 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -308,10 +308,29 @@ class MessageHandler(BaseHandler): room_id=room_id )) + now_token = yield self.hs.get_event_sources().get_current_token() + + limit = pagin_config.limit if pagin_config else None + if limit is None: + limit = 10 + + messages, token = yield self.store.get_recent_events_for_room( + room_id, + limit=limit, + end_token=now_token.room_key, + ) + + start_token = now_token.copy_and_replace("room_key", token[0]) + end_token = now_token.copy_and_replace("room_key", token[1]) + defer.returnValue({ "membership": member_event.membership, "room_id": room_id, - #"messages": messages, + "messages": { + "chunk": [self.hs.serialize_event(m) for m in messages], + "start": start_token.to_string(), + "end": end_token.to_string(), + }, "state": state, #"presence": presence }) diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 98c6af97b..b7d9a2bd0 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -1050,6 +1050,10 @@ class RoomInitialSyncTestCase(RestTestCase): self.assertTrue("m.room.create" in state) + self.assertTrue("messages" in response) + self.assertTrue("chunk" in response["messages"]) + self.assertTrue("end" in response["messages"]) + # (code, response) = yield self.mock_resource.trigger("GET", path, None) # self.assertEquals(200, code, msg=str(response)) # self.assert_dict(json.loads(content), response) From 4c18e08036a15c536070ae6b9c0b314fc032010d Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 18 Nov 2014 15:10:11 +0000 Subject: [PATCH 06/10] Don't expect all _user_cachemap entries to definitely contain a "last_active" key --- synapse/handlers/presence.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 2ccc2245b..36d864e3f 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -180,9 +180,9 @@ class PresenceHandler(BaseHandler): state["presence"] = state.pop("state") if target_user in self._user_cachemap: - state["last_active"] = ( - self._user_cachemap[target_user].get_state()["last_active"] - ) + cached_state = self._user_cachemap[target_user].get_state() + if "last_active" in cached_state: + state["last_active"] = cached_state["last_active"] else: # TODO(paul): Have remote server send us permissions set state = self._get_or_offline_usercache(target_user).get_state() From 759db7d7d51029e23540cffee09b03da2c33a744 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 18 Nov 2014 15:25:55 +0000 Subject: [PATCH 07/10] Added ability to .get_state() from the PresenceHandler by returning a complete m.presence event --- synapse/handlers/presence.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 36d864e3f..325ae4525 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -165,7 +165,7 @@ class PresenceHandler(BaseHandler): defer.returnValue(False) @defer.inlineCallbacks - def get_state(self, target_user, auth_user): + def get_state(self, target_user, auth_user, as_event=False): if target_user.is_mine: visible = yield self.is_presence_visible( observer_user=auth_user, @@ -191,7 +191,20 @@ class PresenceHandler(BaseHandler): state["last_active_ago"] = int( self.clock.time_msec() - state.pop("last_active") ) - defer.returnValue(state) + + if as_event: + content = state + + content["user_id"] = target_user.to_string() + + if "last_active" in content: + content["last_active_ago"] = int( + self._clock.time_msec() - content.pop("last_active") + ) + + defer.returnValue({"type": "m.presence", "content": content}) + else: + defer.returnValue(state) @defer.inlineCallbacks @log_function From 33d328d967341d8819632bc3c5406b8c54f54504 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 18 Nov 2014 15:28:58 +0000 Subject: [PATCH 08/10] Include room members' presence in room initialSync --- synapse/handlers/message.py | 23 ++++++++++++++++++++++- tests/rest/test_rooms.py | 14 ++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 778cdb231..21f9df30e 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -299,6 +299,10 @@ class MessageHandler(BaseHandler): feedback=False): yield self.auth.check_joined_room(room_id, user_id) + # TODO(paul): I wish I was called with user objects not user_id + # strings... + auth_user = self.hs.parse_userid(user_id) + # TODO: These concurrently state_tuples = yield self.store.get_current_state(room_id) state = [self.hs.serialize_event(x) for x in state_tuples] @@ -323,6 +327,23 @@ class MessageHandler(BaseHandler): start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) + room_members = yield self.store.get_room_members(room_id) + + presence_handler = self.hs.get_handlers().presence_handler + presence = [] + for m in room_members: + try: + member_presence = yield presence_handler.get_state( + target_user=self.hs.parse_userid(m.user_id), + auth_user=auth_user, + as_event=True, + ) + presence.append(member_presence) + except Exception as e: + logger.exception("Failed to get member presence of %r", + m.user_id + ) + defer.returnValue({ "membership": member_event.membership, "room_id": room_id, @@ -332,5 +353,5 @@ class MessageHandler(BaseHandler): "end": end_token.to_string(), }, "state": state, - #"presence": presence + "presence": presence }) diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index b7d9a2bd0..ad3631d51 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -1026,6 +1026,12 @@ class RoomInitialSyncTestCase(RestTestCase): synapse.rest.room.register_servlets(hs, self.mock_resource) + # Since I'm getting my own presence I need to exist as far as presence + # is concerned. + hs.get_handlers().presence_handler.registered_user( + hs.parse_userid(self.user_id) + ) + # create the room self.room_id = yield self.create_room_as(self.user_id) @@ -1054,6 +1060,14 @@ class RoomInitialSyncTestCase(RestTestCase): self.assertTrue("chunk" in response["messages"]) self.assertTrue("end" in response["messages"]) + self.assertTrue("presence" in response) + + presence_by_user = {e["content"]["user_id"]: e + for e in response["presence"] + } + self.assertTrue(self.user_id in presence_by_user) + self.assertEquals("m.presence", presence_by_user[self.user_id]["type"]) + # (code, response) = yield self.mock_resource.trigger("GET", path, None) # self.assertEquals(200, code, msg=str(response)) # self.assert_dict(json.loads(content), response) From 75e95c45a2da90528261c028e97e60a493106c5a Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 18 Nov 2014 16:02:44 +0000 Subject: [PATCH 09/10] Rename message handler's new snapshot_room to room_initial_sync() as that better suits its purpose --- synapse/handlers/message.py | 2 +- synapse/rest/room.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 21f9df30e..081030dbb 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -295,7 +295,7 @@ class MessageHandler(BaseHandler): defer.returnValue(ret) @defer.inlineCallbacks - def snapshot_room(self, user_id, room_id, pagin_config=None, + def room_initial_sync(self, user_id, room_id, pagin_config=None, feedback=False): yield self.auth.check_joined_room(room_id, user_id) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 393d6284f..e656dd3f3 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -362,7 +362,7 @@ class RoomInitialSyncRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): user = yield self.auth.get_user_by_req(request) - events = yield self.handlers.message_handler.snapshot_room( + events = yield self.handlers.message_handler.room_initial_sync( room_id=room_id, user_id=user.to_string(), ) From 88dfa7baa6b5ed995954021328ecc4927693504a Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 18 Nov 2014 16:34:43 +0000 Subject: [PATCH 10/10] Ensure to parse a real pagination config object out of room initialSync request and pass it on --- synapse/rest/room.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index e656dd3f3..17322ee9b 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -362,11 +362,13 @@ class RoomInitialSyncRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): user = yield self.auth.get_user_by_req(request) - events = yield self.handlers.message_handler.room_initial_sync( + pagination_config = PaginationConfig.from_request(request) + content = yield self.handlers.message_handler.room_initial_sync( room_id=room_id, user_id=user.to_string(), + pagin_config=pagination_config, ) - defer.returnValue((200, events)) + defer.returnValue((200, content)) class RoomTriggerBackfill(RestServlet):