Merge branch 'develop' of github.com:matrix-org/synapse into erikj/state_storage

synapse/app/synchrotron.py

@@ -399,6 +399,9 @@ class SynchrotronServer(HomeServer):
             notify_from_stream(
                 result, "typing", "typing_key", room="room_id"
             )
+            notify_from_stream(
+                result, "to_device", "to_device_key", user="user_id"
+            )
 
         while True:
             try:

synapse/federation/federation_server.py

@@ -223,16 +223,14 @@ class FederationServer(FederationBase):
         if not in_room:
             raise AuthError(403, "Host not in room.")
 
-        pdus = yield self.handler.get_state_for_pdu(
+        state_ids = yield self.handler.get_state_ids_for_pdu(
             room_id, event_id,
         )
-        auth_chain = yield self.store.get_auth_chain(
+        auth_chain_ids = yield self.store.get_auth_chain_ids(state_ids)
-            [pdu.event_id for pdu in pdus]
-        )
 
         defer.returnValue((200, {
-            "pdu_ids": [pdu.event_id for pdu in pdus],
+            "pdu_ids": state_ids,
-            "auth_chain_ids": [pdu.event_id for pdu in auth_chain],
+            "auth_chain_ids": auth_chain_ids,
         }))
 
     @defer.inlineCallbacks

synapse/handlers/federation.py

@@ -101,6 +101,9 @@ class FederationHandler(BaseHandler):
     def on_receive_pdu(self, origin, pdu, state=None, auth_chain=None):
         """ Called by the ReplicationLayer when we have a new pdu. We need to
         do auth checks and put it through the StateHandler.
+
+        auth_chain and state are None if we already have the necessary state
+        and prev_events in the db
         """
         event = pdu
 
@@ -118,12 +121,21 @@ class FederationHandler(BaseHandler):
 
         # FIXME (erikj): Awful hack to make the case where we are not currently
         # in the room work
-        is_in_room = yield self.auth.check_host_in_room(
+        # If state and auth_chain are None, then we don't need to do this check
-            event.room_id,
+        # as we already know we have enough state in the DB to handle this
-            self.server_name
+        # event.
-        )
+        if state and auth_chain and not event.internal_metadata.is_outlier():
-        if not is_in_room and not event.internal_metadata.is_outlier():
+            is_in_room = yield self.auth.check_host_in_room(
-            logger.debug("Got event for room we're not in.")
+                event.room_id,
+                self.server_name
+            )
+        else:
+            is_in_room = True
+        if not is_in_room:
+            logger.info(
+                "Got event for room we're not in: %r %r",
+                event.room_id, event.event_id
+            )
 
             try:
                 event_stream_id, max_stream_id = yield self._persist_auth_tree(
@@ -1062,6 +1074,8 @@ class FederationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def get_state_for_pdu(self, room_id, event_id):
+        """Returns the state at the event. i.e. not including said event.
+        """
         yield run_on_reactor()
 
         state_groups = yield self.store.get_state_groups(
@@ -1102,6 +1116,34 @@ class FederationHandler(BaseHandler):
         else:
             defer.returnValue([])
 
+    @defer.inlineCallbacks
+    def get_state_ids_for_pdu(self, room_id, event_id):
+        """Returns the state at the event. i.e. not including said event.
+        """
+        yield run_on_reactor()
+
+        state_groups = yield self.store.get_state_groups_ids(
+            room_id, [event_id]
+        )
+
+        if state_groups:
+            _, state = state_groups.items().pop()
+            results = state
+
+            event = yield self.store.get_event(event_id)
+            if event and event.is_state():
+                # Get previous state
+                if "replaces_state" in event.unsigned:
+                    prev_id = event.unsigned["replaces_state"]
+                    if prev_id != event.event_id:
+                        results[(event.type, event.state_key)] = prev_id
+                else:
+                    del results[(event.type, event.state_key)]
+
+            defer.returnValue(results.values())
+        else:
+            defer.returnValue([])
+
     @defer.inlineCallbacks
     @log_function
     def on_backfill_request(self, origin, room_id, pdu_list, limit):

synapse/notifier.py

@@ -423,7 +423,8 @@ class Notifier(object):
     def _is_world_readable(self, room_id):
         state = yield self.state_handler.get_current_state(
             room_id,
-            EventTypes.RoomHistoryVisibility
+            EventTypes.RoomHistoryVisibility,
+            "",
         )
         if state and "history_visibility" in state.content:
             defer.returnValue(state.content["history_visibility"] == "world_readable")

synapse/rest/client/v2_alpha/sendtodevice.py

@@ -78,7 +78,7 @@ class SendToDeviceRestServlet(servlet.RestServlet):
         stream_id = yield self.store.add_messages_to_device_inbox(local_messages)
 
         self.notifier.on_new_event(
-            "to_device", stream_id, users=local_messages.keys()
+            "to_device_key", stream_id, users=local_messages.keys()
         )
 
         response = (200, {})

synapse/storage/transactions.py

@@ -245,7 +245,7 @@ class TransactionStore(SQLBaseStore):
 
         return self.cursor_to_dict(txn)
 
-    @cached()
+    @cached(max_entries=10000)
     def get_destination_retry_timings(self, destination):
         """Gets the current retry timings (if any) for a given destination.