Merge branch 'develop' of github.com:matrix-org/synapse into matrix-org-hotfixes

CHANGES.rst

@@ -1,3 +1,9 @@
+Changes in synapse v0.18.7 (2017-01-09)
+=======================================
+
+No changes from v0.18.7-rc2
+
+
 Changes in synapse v0.18.7-rc2 (2017-01-07)
 ===========================================
 
@@ -6,6 +12,7 @@ Bug fixes:
 * Fix error in rc1's discarding invalid inbound traffic logic that was
   incorrectly discarding missing events
 
+
 Changes in synapse v0.18.7-rc1 (2017-01-06)
 ===========================================
 
@@ -16,6 +23,7 @@ Bug fixes:
 * Discard inbound federation traffic from invalid domains, to immunise
   against #1753
 
+
 Changes in synapse v0.18.6 (2017-01-06)
 =======================================
 
@@ -23,6 +31,7 @@ Bug fixes:
 
 * Fix bug when checking if a guest user is allowed to join a room (PR #1772)
 
+
 Changes in synapse v0.18.6-rc3 (2017-01-05)
 ===========================================
 
@@ -38,6 +47,7 @@ Changes:
 
 * Improve logging for debugging deadlocks (PR #1766, PR #1767)
 
+
 Changes in synapse v0.18.6-rc2 (2016-12-30)
 ===========================================
 
@@ -55,6 +65,7 @@ Bug fixes:
 
 * Make sure that outbound connections are closed (PR #1725)
 
+
 Changes in synapse v0.18.5 (2016-12-16)
 =======================================
 

README.rst

@@ -658,7 +658,7 @@ configuration might look like::
       }
   }
 
-You will also want to set ``bind_address: 127.0.0.1`` and ``x_forwarded: true``
+You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
 for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
 recorded correctly.
 

synapse/__init__.py

@@ -16,4 +16,4 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-__version__ = "0.18.7-rc2"
+__version__ = "0.18.7"

synapse/app/appservice.py

@@ -76,7 +76,8 @@ class AppserviceServer(HomeServer):
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
-        bind_address = listener_config.get("bind_address", "")
+        bind_address = listener_config.get("bind_address", None)
+        bind_addresses = listener_config.get("bind_addresses", [])
         site_tag = listener_config.get("tag", port)
         resources = {}
         for res in listener_config["resources"]:
@@ -85,6 +86,11 @@ class AppserviceServer(HomeServer):
                     resources[METRICS_PREFIX] = MetricsResource(self)
 
         root_resource = create_resource_tree(resources, Resource())
+
+        if bind_address is not None:
+            bind_addresses.append(bind_address)
+
+        for address in bind_addresses:
             reactor.listenTCP(
                 port,
                 SynapseSite(
@@ -93,8 +99,9 @@ class AppserviceServer(HomeServer):
                     listener_config,
                     root_resource,
                 ),
-            interface=bind_address
+                interface=address
             )
+
         logger.info("Synapse appservice now listening on port %d", port)
 
     def start_listening(self, listeners):
@@ -102,6 +109,13 @@ class AppserviceServer(HomeServer):
             if listener["type"] == "http":
                 self._listen_http(listener)
             elif listener["type"] == "manhole":
+                bind_address = listener.get("bind_address", None)
+                bind_addresses = listener.get("bind_addresses", [])
+
+                if bind_address is not None:
+                    bind_addresses.append(bind_address)
+
+                for address in bind_addresses:
                     reactor.listenTCP(
                         listener["port"],
                         manhole(
@@ -109,7 +123,7 @@ class AppserviceServer(HomeServer):
                             password="rabbithole",
                             globals={"hs": self},
                         ),
-                    interface=listener.get("bind_address", '127.0.0.1')
+                        interface=address
                     )
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])

synapse/app/client_reader.py

@@ -90,7 +90,8 @@ class ClientReaderServer(HomeServer):
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
-        bind_address = listener_config.get("bind_address", "")
+        bind_address = listener_config.get("bind_address", None)
+        bind_addresses = listener_config.get("bind_addresses", [])
         site_tag = listener_config.get("tag", port)
         resources = {}
         for res in listener_config["resources"]:
@@ -108,6 +109,11 @@ class ClientReaderServer(HomeServer):
                     })
 
         root_resource = create_resource_tree(resources, Resource())
+
+        if bind_address is not None:
+            bind_addresses.append(bind_address)
+
+        for address in bind_addresses:
             reactor.listenTCP(
                 port,
                 SynapseSite(
@@ -116,8 +122,9 @@ class ClientReaderServer(HomeServer):
                     listener_config,
                     root_resource,
                 ),
-            interface=bind_address
+                interface=address
             )
+
         logger.info("Synapse client reader now listening on port %d", port)
 
     def start_listening(self, listeners):
@@ -125,6 +132,13 @@ class ClientReaderServer(HomeServer):
             if listener["type"] == "http":
                 self._listen_http(listener)
             elif listener["type"] == "manhole":
+                bind_address = listener.get("bind_address", None)
+                bind_addresses = listener.get("bind_addresses", [])
+
+                if bind_address is not None:
+                    bind_addresses.append(bind_address)
+
+                for address in bind_addresses:
                     reactor.listenTCP(
                         listener["port"],
                         manhole(
@@ -132,7 +146,7 @@ class ClientReaderServer(HomeServer):
                             password="rabbithole",
                             globals={"hs": self},
                         ),
-                    interface=listener.get("bind_address", '127.0.0.1')
+                        interface=address
                     )
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])

synapse/app/federation_reader.py

@@ -86,7 +86,8 @@ class FederationReaderServer(HomeServer):
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
-        bind_address = listener_config.get("bind_address", "")
+        bind_address = listener_config.get("bind_address", None)
+        bind_addresses = listener_config.get("bind_addresses", [])
         site_tag = listener_config.get("tag", port)
         resources = {}
         for res in listener_config["resources"]:
@@ -99,6 +100,11 @@ class FederationReaderServer(HomeServer):
                     })
 
         root_resource = create_resource_tree(resources, Resource())
+
+        if bind_address is not None:
+            bind_addresses.append(bind_address)
+
+        for address in bind_addresses:
             reactor.listenTCP(
                 port,
                 SynapseSite(
@@ -107,8 +113,9 @@ class FederationReaderServer(HomeServer):
                     listener_config,
                     root_resource,
                 ),
-            interface=bind_address
+                interface=address
             )
+
         logger.info("Synapse federation reader now listening on port %d", port)
 
     def start_listening(self, listeners):
@@ -116,6 +123,13 @@ class FederationReaderServer(HomeServer):
             if listener["type"] == "http":
                 self._listen_http(listener)
             elif listener["type"] == "manhole":
+                bind_address = listener.get("bind_address", None)
+                bind_addresses = listener.get("bind_addresses", [])
+
+                if bind_address is not None:
+                    bind_addresses.append(bind_address)
+
+                for address in bind_addresses:
                     reactor.listenTCP(
                         listener["port"],
                         manhole(
@@ -123,7 +137,7 @@ class FederationReaderServer(HomeServer):
                             password="rabbithole",
                             globals={"hs": self},
                         ),
-                    interface=listener.get("bind_address", '127.0.0.1')
+                        interface=address
                     )
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])

synapse/app/federation_sender.py

@@ -82,7 +82,8 @@ class FederationSenderServer(HomeServer):
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
-        bind_address = listener_config.get("bind_address", "")
+        bind_address = listener_config.get("bind_address", None)
+        bind_addresses = listener_config.get("bind_addresses", [])
         site_tag = listener_config.get("tag", port)
         resources = {}
         for res in listener_config["resources"]:
@@ -91,6 +92,11 @@ class FederationSenderServer(HomeServer):
                     resources[METRICS_PREFIX] = MetricsResource(self)
 
         root_resource = create_resource_tree(resources, Resource())
+
+        if bind_address is not None:
+            bind_addresses.append(bind_address)
+
+        for address in bind_addresses:
             reactor.listenTCP(
                 port,
                 SynapseSite(
@@ -99,8 +105,9 @@ class FederationSenderServer(HomeServer):
                     listener_config,
                     root_resource,
                 ),
-            interface=bind_address
+                interface=address
             )
+
         logger.info("Synapse federation_sender now listening on port %d", port)
 
     def start_listening(self, listeners):
@@ -108,6 +115,13 @@ class FederationSenderServer(HomeServer):
             if listener["type"] == "http":
                 self._listen_http(listener)
             elif listener["type"] == "manhole":
+                bind_address = listener.get("bind_address", None)
+                bind_addresses = listener.get("bind_addresses", [])
+
+                if bind_address is not None:
+                    bind_addresses.append(bind_address)
+
+                for address in bind_addresses:
                     reactor.listenTCP(
                         listener["port"],
                         manhole(
@@ -115,7 +129,7 @@ class FederationSenderServer(HomeServer):
                             password="rabbithole",
                             globals={"hs": self},
                         ),
-                    interface=listener.get("bind_address", '127.0.0.1')
+                        interface=address
                     )
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])

synapse/app/homeserver.py

@@ -107,7 +107,8 @@ def build_resource_for_web_client(hs):
 class SynapseHomeServer(HomeServer):
     def _listener_http(self, config, listener_config):
         port = listener_config["port"]
-        bind_address = listener_config.get("bind_address", "")
+        bind_address = listener_config.get("bind_address", None)
+        bind_addresses = listener_config.get("bind_addresses", [])
         tls = listener_config.get("tls", False)
         site_tag = listener_config.get("tag", port)
 
@@ -173,7 +174,12 @@ class SynapseHomeServer(HomeServer):
             root_resource = Resource()
 
         root_resource = create_resource_tree(resources, root_resource)
+
+        if bind_address is not None:
+            bind_addresses.append(bind_address)
+
         if tls:
+            for address in bind_addresses:
                 reactor.listenSSL(
                     port,
                     SynapseSite(
@@ -183,9 +189,10 @@ class SynapseHomeServer(HomeServer):
                         root_resource,
                     ),
                     self.tls_server_context_factory,
-                interface=bind_address
+                    interface=address
                 )
         else:
+            for address in bind_addresses:
                 reactor.listenTCP(
                     port,
                     SynapseSite(
@@ -194,7 +201,7 @@ class SynapseHomeServer(HomeServer):
                         listener_config,
                         root_resource,
                     ),
-                interface=bind_address
+                    interface=address
                 )
         logger.info("Synapse now listening on port %d", port)
 
@@ -205,6 +212,13 @@ class SynapseHomeServer(HomeServer):
             if listener["type"] == "http":
                 self._listener_http(config, listener)
             elif listener["type"] == "manhole":
+                bind_address = listener.get("bind_address", None)
+                bind_addresses = listener.get("bind_addresses", [])
+
+                if bind_address is not None:
+                    bind_addresses.append(bind_address)
+
+                for address in bind_addresses:
                     reactor.listenTCP(
                         listener["port"],
                         manhole(
@@ -212,7 +226,7 @@ class SynapseHomeServer(HomeServer):
                             password="rabbithole",
                             globals={"hs": self},
                         ),
-                    interface=listener.get("bind_address", '127.0.0.1')
+                        interface=address
                     )
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])

synapse/app/media_repository.py

@@ -87,7 +87,8 @@ class MediaRepositoryServer(HomeServer):
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
-        bind_address = listener_config.get("bind_address", "")
+        bind_address = listener_config.get("bind_address", None)
+        bind_addresses = listener_config.get("bind_addresses", [])
         site_tag = listener_config.get("tag", port)
         resources = {}
         for res in listener_config["resources"]:
@@ -105,6 +106,11 @@ class MediaRepositoryServer(HomeServer):
                     })
 
         root_resource = create_resource_tree(resources, Resource())
+
+        if bind_address is not None:
+            bind_addresses.append(bind_address)
+
+        for address in bind_addresses:
             reactor.listenTCP(
                 port,
                 SynapseSite(
@@ -113,8 +119,9 @@ class MediaRepositoryServer(HomeServer):
                     listener_config,
                     root_resource,
                 ),
-            interface=bind_address
+                interface=address
             )
+
         logger.info("Synapse media repository now listening on port %d", port)
 
     def start_listening(self, listeners):
@@ -122,6 +129,13 @@ class MediaRepositoryServer(HomeServer):
             if listener["type"] == "http":
                 self._listen_http(listener)
             elif listener["type"] == "manhole":
+                bind_address = listener.get("bind_address", None)
+                bind_addresses = listener.get("bind_addresses", [])
+
+                if bind_address is not None:
+                    bind_addresses.append(bind_address)
+
+                for address in bind_addresses:
                     reactor.listenTCP(
                         listener["port"],
                         manhole(
@@ -129,7 +143,7 @@ class MediaRepositoryServer(HomeServer):
                             password="rabbithole",
                             globals={"hs": self},
                         ),
-                    interface=listener.get("bind_address", '127.0.0.1')
+                        interface=address
                     )
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])

synapse/app/pusher.py

@@ -121,7 +121,8 @@ class PusherServer(HomeServer):
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
-        bind_address = listener_config.get("bind_address", "")
+        bind_address = listener_config.get("bind_address", None)
+        bind_addresses = listener_config.get("bind_addresses", [])
         site_tag = listener_config.get("tag", port)
         resources = {}
         for res in listener_config["resources"]:
@@ -130,6 +131,22 @@ class PusherServer(HomeServer):
                     resources[METRICS_PREFIX] = MetricsResource(self)
 
         root_resource = create_resource_tree(resources, Resource())
+
+        if bind_address is not None:
+            bind_addresses.append(bind_address)
+
+        for address in bind_addresses:
+            reactor.listenTCP(
+                port,
+                SynapseSite(
+                    "synapse.access.http.%s" % (site_tag,),
+                    site_tag,
+                    listener_config,
+                    root_resource,
+                ),
+                interface=address
+            )
+        else:
             reactor.listenTCP(
                 port,
                 SynapseSite(
@@ -140,6 +157,7 @@ class PusherServer(HomeServer):
                 ),
                 interface=bind_address
             )
+
         logger.info("Synapse pusher now listening on port %d", port)
 
     def start_listening(self, listeners):
@@ -147,6 +165,13 @@ class PusherServer(HomeServer):
             if listener["type"] == "http":
                 self._listen_http(listener)
             elif listener["type"] == "manhole":
+                bind_address = listener.get("bind_address", None)
+                bind_addresses = listener.get("bind_addresses", [])
+
+                if bind_address is not None:
+                    bind_addresses.append(bind_address)
+
+                for address in bind_addresses:
                     reactor.listenTCP(
                         listener["port"],
                         manhole(
@@ -154,7 +179,7 @@ class PusherServer(HomeServer):
                             password="rabbithole",
                             globals={"hs": self},
                         ),
-                    interface=listener.get("bind_address", '127.0.0.1')
+                        interface=address
                     )
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])

synapse/app/synchrotron.py

@@ -289,7 +289,8 @@ class SynchrotronServer(HomeServer):
 
     def _listen_http(self, listener_config):
         port = listener_config["port"]
-        bind_address = listener_config.get("bind_address", "")
+        bind_address = listener_config.get("bind_address", None)
+        bind_addresses = listener_config.get("bind_addresses", [])
         site_tag = listener_config.get("tag", port)
         resources = {}
         for res in listener_config["resources"]:
@@ -310,6 +311,11 @@ class SynchrotronServer(HomeServer):
                     })
 
         root_resource = create_resource_tree(resources, Resource())
+
+        if bind_address is not None:
+            bind_addresses.append(bind_address)
+
+        for address in bind_addresses:
             reactor.listenTCP(
                 port,
                 SynapseSite(
@@ -318,8 +324,9 @@ class SynchrotronServer(HomeServer):
                     listener_config,
                     root_resource,
                 ),
-            interface=bind_address
+                interface=address
             )
+
         logger.info("Synapse synchrotron now listening on port %d", port)
 
     def start_listening(self, listeners):
@@ -327,6 +334,13 @@ class SynchrotronServer(HomeServer):
             if listener["type"] == "http":
                 self._listen_http(listener)
             elif listener["type"] == "manhole":
+                bind_address = listener.get("bind_address", None)
+                bind_addresses = listener.get("bind_addresses", [])
+
+                if bind_address is not None:
+                    bind_addresses.append(bind_address)
+
+                for address in bind_addresses:
                     reactor.listenTCP(
                         listener["port"],
                         manhole(
@@ -334,7 +348,7 @@ class SynchrotronServer(HomeServer):
                             password="rabbithole",
                             globals={"hs": self},
                         ),
-                    interface=listener.get("bind_address", '127.0.0.1')
+                        interface=address
                     )
             else:
                 logger.warn("Unrecognized listener type: %s", listener["type"])

synapse/config/server.py

@@ -155,9 +155,14 @@ class ServerConfig(Config):
             # The port to listen for HTTPS requests on.
             port: %(bind_port)s
 
-            # Local interface to listen on.
+            # Local addresses to listen on.
-            # The empty string will cause synapse to listen on all interfaces.
+            # This will listen on all IPv4 addresses by default.
-            bind_address: ''
+            bind_addresses:
+              - '0.0.0.0'
+              # Uncomment to listen on all IPv6 interfaces
+              # N.B: On at least Linux this will also listen on all IPv4
+              # addresses, so you will need to comment out the line above.
+              # - '::'
 
             # This is a 'http' listener, allows us to specify 'resources'.
             type: http
@@ -188,7 +193,7 @@ class ServerConfig(Config):
           # For when matrix traffic passes through loadbalancer that unwraps TLS.
           - port: %(unsecure_port)s
             tls: false
-            bind_address: ''
+            bind_addresses: ['0.0.0.0']
             type: http
 
             x_forwarded: false

synapse/federation/federation_server.py

@@ -52,8 +52,8 @@ class FederationServer(FederationBase):
 
         self.auth = hs.get_auth()
 
-        self._room_pdu_linearizer = Linearizer()
+        self._room_pdu_linearizer = Linearizer("fed_room_pdu")
-        self._server_linearizer = Linearizer()
+        self._server_linearizer = Linearizer("fed_server")
 
         # We cache responses to state queries, as they take a while and often
         # come in waves.

synapse/handlers/auth.py

@@ -607,7 +607,7 @@ class AuthHandler(BaseHandler):
         # types (mediums) of threepid. For now, we still use the existing
         # infrastructure, but this is the start of synapse gaining knowledge
         # of specific types of threepid (and fixes the fact that checking
-        # for the presenc eof an email address during password reset was
+        # for the presence of an email address during password reset was
         # case sensitive).
         if medium == 'email':
             address = address.lower()
@@ -617,6 +617,17 @@ class AuthHandler(BaseHandler):
             self.hs.get_clock().time_msec()
         )
 
+    @defer.inlineCallbacks
+    def delete_threepid(self, user_id, medium, address):
+        # 'Canonicalise' email addresses as per above
+        if medium == 'email':
+            address = address.lower()
+
+        ret = yield self.store.user_delete_threepid(
+            user_id, medium, address,
+        )
+        defer.returnValue(ret)
+
     def _save_session(self, session):
         # TODO: Persistent storage
         logger.debug("Saving session %s", session)
@@ -656,8 +667,8 @@ class AuthHandler(BaseHandler):
             Whether self.hash(password) == stored_hash (bool).
         """
         if stored_hash:
-            return bcrypt.hashpw(password + self.hs.config.password_pepper,
+            return bcrypt.hashpw(password.encode('utf8') + self.hs.config.password_pepper,
-                                 stored_hash.encode('utf-8')) == stored_hash
+                                 stored_hash.encode('utf8')) == stored_hash
         else:
             return False
 

synapse/handlers/message.py

@@ -279,7 +279,9 @@ class MessageHandler(BaseHandler):
 
         if event.type == EventTypes.Message:
             presence = self.hs.get_presence_handler()
-            yield presence.bump_presence_active_time(user)
+            # We don't want to block sending messages on any presence code. This
+            # matters as sometimes presence code can take a while.
+            preserve_fn(presence.bump_presence_active_time)(user)
 
     @defer.inlineCallbacks
     def deduplicate_state_event(self, event, context):

synapse/handlers/room_list.py

@@ -69,11 +69,14 @@ class RoomListHandler(BaseHandler):
                 limit, since_token, search_filter, network_tuple=network_tuple,
             )
 
-        result = self.response_cache.get((limit, since_token, network_tuple))
+        key = (limit, since_token, network_tuple)
+        result = self.response_cache.get(key)
         if not result:
             result = self.response_cache.set(
-                (limit, since_token, network_tuple),
+                key,
-                self._get_public_room_list(limit, since_token, network_tuple=network_tuple)
+                self._get_public_room_list(
+                    limit, since_token, network_tuple=network_tuple
+                )
             )
         return result
 

synapse/handlers/room_member.py

@@ -45,7 +45,7 @@ class RoomMemberHandler(BaseHandler):
     def __init__(self, hs):
         super(RoomMemberHandler, self).__init__(hs)
 
-        self.member_linearizer = Linearizer()
+        self.member_linearizer = Linearizer(name="member")
 
         self.clock = hs.get_clock()
 
@@ -89,7 +89,7 @@ class RoomMemberHandler(BaseHandler):
         duplicate = yield msg_handler.deduplicate_state_event(event, context)
         if duplicate is not None:
             # Discard the new event since this membership change is a no-op.
-            return
+            defer.returnValue(duplicate)
 
         yield msg_handler.handle_new_client_event(
             requester,
@@ -120,6 +120,8 @@ class RoomMemberHandler(BaseHandler):
                 if prev_member_event.membership == Membership.JOIN:
                     user_left_room(self.distributor, target, room_id)
 
+        defer.returnValue(event)
+
     @defer.inlineCallbacks
     def remote_join(self, remote_room_hosts, room_id, user, content):
         if len(remote_room_hosts) == 0:
@@ -187,6 +189,7 @@ class RoomMemberHandler(BaseHandler):
             ratelimit=True,
             content=None,
     ):
+        content_specified = bool(content)
         if content is None:
             content = {}
 
@@ -229,6 +232,12 @@ class RoomMemberHandler(BaseHandler):
                     errcode=Codes.BAD_STATE
                 )
 
+            same_content = content == old_state.content
+            same_membership = old_membership == effective_membership_state
+            same_sender = requester.user.to_string() == old_state.sender
+            if same_sender and same_membership and same_content:
+                defer.returnValue(old_state)
+
         is_host_in_room = yield self._is_host_in_room(current_state_ids)
 
         if effective_membership_state == Membership.JOIN:
@@ -247,6 +256,7 @@ class RoomMemberHandler(BaseHandler):
                 content["membership"] = Membership.JOIN
 
                 profile = self.hs.get_handlers().profile_handler
+                if not content_specified:
                     content["displayname"] = yield profile.get_displayname(target)
                     content["avatar_url"] = yield profile.get_avatar_url(target)
 
@@ -290,7 +300,7 @@ class RoomMemberHandler(BaseHandler):
 
                         defer.returnValue({})
 
-        yield self._local_membership_update(
+        res = yield self._local_membership_update(
             requester=requester,
             target=target,
             room_id=room_id,
@@ -300,6 +310,7 @@ class RoomMemberHandler(BaseHandler):
             prev_event_ids=latest_event_ids,
             content=content,
         )
+        defer.returnValue(res)
 
     @defer.inlineCallbacks
     def send_membership_event(

synapse/http/client.py

@@ -25,7 +25,7 @@ from synapse.http.endpoint import SpiderEndpoint
 from canonicaljson import encode_canonical_json
 
 from twisted.internet import defer, reactor, ssl, protocol, task
-from twisted.internet.endpoints import SSL4ClientEndpoint, TCP4ClientEndpoint
+from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
 from twisted.web.client import (
     BrowserLikeRedirectAgent, ContentDecoderAgent, GzipDecoder, Agent,
     readBody, PartialDownloadError,
@@ -386,26 +386,23 @@ class SpiderEndpointFactory(object):
 
     def endpointForURI(self, uri):
         logger.info("Getting endpoint for %s", uri.toBytes())
+
         if uri.scheme == "http":
-            return SpiderEndpoint(
+            endpoint_factory = HostnameEndpoint
-                reactor, uri.host, uri.port, self.blacklist, self.whitelist,
-                endpoint=TCP4ClientEndpoint,
-                endpoint_kw_args={
-                    'timeout': 15
-                },
-            )
         elif uri.scheme == "https":
-            tlsPolicy = self.policyForHTTPS.creatorForNetloc(uri.host, uri.port)
+            tlsCreator = self.policyForHTTPS.creatorForNetloc(uri.host, uri.port)
-            return SpiderEndpoint(
+
-                reactor, uri.host, uri.port, self.blacklist, self.whitelist,
+            def endpoint_factory(reactor, host, port, **kw):
-                endpoint=SSL4ClientEndpoint,
+                return wrapClientTLS(
-                endpoint_kw_args={
+                    tlsCreator,
-                    'sslContextFactory': tlsPolicy,
+                    HostnameEndpoint(reactor, host, port, **kw))
-                    'timeout': 15
-                },
-            )
         else:
             logger.warn("Can't get endpoint for unrecognised scheme %s", uri.scheme)
+            return None
+        return SpiderEndpoint(
+            reactor, uri.host, uri.port, self.blacklist, self.whitelist,
+            endpoint=endpoint_factory, endpoint_kw_args=dict(timeout=15),
+        )
 
 
 class SpiderHttpClient(SimpleHttpClient):

synapse/http/endpoint.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from twisted.internet.endpoints import SSL4ClientEndpoint, TCP4ClientEndpoint
+from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
 from twisted.internet import defer, reactor
 from twisted.internet.error import ConnectError
 from twisted.names import client, dns
@@ -58,11 +58,13 @@ def matrix_federation_endpoint(reactor, destination, ssl_context_factory=None,
         endpoint_kw_args.update(timeout=timeout)
 
     if ssl_context_factory is None:
-        transport_endpoint = TCP4ClientEndpoint
+        transport_endpoint = HostnameEndpoint
         default_port = 8008
     else:
-        transport_endpoint = SSL4ClientEndpoint
+        def transport_endpoint(reactor, host, port, timeout):
-        endpoint_kw_args.update(sslContextFactory=ssl_context_factory)
+            return wrapClientTLS(
+                ssl_context_factory,
+                HostnameEndpoint(reactor, host, port, timeout=timeout))
         default_port = 8448
 
     if port is None:
@@ -142,7 +144,7 @@ class SpiderEndpoint(object):
     Implements twisted.internet.interfaces.IStreamClientEndpoint.
     """
     def __init__(self, reactor, host, port, blacklist, whitelist,
-                 endpoint=TCP4ClientEndpoint, endpoint_kw_args={}):
+                 endpoint=HostnameEndpoint, endpoint_kw_args={}):
         self.reactor = reactor
         self.host = host
         self.port = port
@@ -180,7 +182,7 @@ class SRVClientEndpoint(object):
     """
 
     def __init__(self, reactor, service, domain, protocol="tcp",
-                 default_port=None, endpoint=TCP4ClientEndpoint,
+                 default_port=None, endpoint=HostnameEndpoint,
                  endpoint_kw_args={}):
         self.reactor = reactor
         self.service_name = "_%s._%s.%s" % (service, protocol, domain)

synapse/python_dependencies.py

@@ -24,7 +24,7 @@ REQUIREMENTS = {
     "signedjson>=1.0.0": ["signedjson>=1.0.0"],
     "pynacl==0.3.0": ["nacl==0.3.0", "nacl.bindings"],
     "service_identity>=1.0.0": ["service_identity>=1.0.0"],
-    "Twisted>=15.1.0": ["twisted>=15.1.0"],
+    "Twisted>=16.0.0": ["twisted>=16.0.0"],
     "pyopenssl>=0.14": ["OpenSSL>=0.14"],
     "pyyaml": ["yaml"],
     "pyasn1": ["pyasn1"],

synapse/rest/client/v1/room.py

@@ -152,6 +152,16 @@ class RoomStateEventRestServlet(ClientV1RestServlet):
         if state_key is not None:
             event_dict["state_key"] = state_key
 
+        if event_type == EventTypes.Member:
+            membership = content.get("membership", None)
+            event = yield self.handlers.room_member_handler.update_membership(
+                requester,
+                target=UserID.from_string(state_key),
+                room_id=room_id,
+                action=membership,
+                content=content,
+            )
+        else:
             msg_handler = self.handlers.message_handler
             event, context = yield msg_handler.create_event(
                 event_dict,
@@ -159,16 +169,12 @@ class RoomStateEventRestServlet(ClientV1RestServlet):
                 txn_id=txn_id,
             )
 
-        if event_type == EventTypes.Member:
-            yield self.handlers.room_member_handler.send_membership_event(
-                requester,
-                event,
-                context,
-            )
-        else:
             yield msg_handler.send_nonmember_event(requester, event, context)
 
-        defer.returnValue((200, {"event_id": event.event_id}))
+        ret = {}
+        if event:
+            ret = {"event_id": event.event_id}
+        defer.returnValue((200, ret))
 
 
 # TODO: Needs unit testing for generic events + feedback

synapse/rest/client/v2_alpha/account.py

@@ -241,7 +241,7 @@ class ThreepidRestServlet(RestServlet):
 
         for reqd in ['medium', 'address', 'validated_at']:
             if reqd not in threepid:
-                logger.warn("Couldn't add 3pid: invalid response from ID sevrer")
+                logger.warn("Couldn't add 3pid: invalid response from ID server")
                 raise SynapseError(500, "Invalid response from ID Server")
 
         yield self.auth_handler.add_threepid(
@@ -263,9 +263,43 @@ class ThreepidRestServlet(RestServlet):
         defer.returnValue((200, {}))
 
 
+class ThreepidDeleteRestServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/account/3pid/delete$", releases=())
+
+    def __init__(self, hs):
+        super(ThreepidDeleteRestServlet, self).__init__()
+        self.auth = hs.get_auth()
+        self.auth_handler = hs.get_auth_handler()
+
+    @defer.inlineCallbacks
+    def on_POST(self, request):
+        yield run_on_reactor()
+
+        body = parse_json_object_from_request(request)
+
+        required = ['medium', 'address']
+        absent = []
+        for k in required:
+            if k not in body:
+                absent.append(k)
+
+        if absent:
+            raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM)
+
+        requester = yield self.auth.get_user_by_req(request)
+        user_id = requester.user.to_string()
+
+        yield self.auth_handler.delete_threepid(
+            user_id, body['medium'], body['address']
+        )
+
+        defer.returnValue((200, {}))
+
+
 def register_servlets(hs, http_server):
     PasswordRequestTokenRestServlet(hs).register(http_server)
     PasswordRestServlet(hs).register(http_server)
     DeactivateAccountRestServlet(hs).register(http_server)
     ThreepidRequestTokenRestServlet(hs).register(http_server)
     ThreepidRestServlet(hs).register(http_server)
+    ThreepidDeleteRestServlet(hs).register(http_server)

synapse/rest/media/v1/media_repository.py

@@ -61,7 +61,7 @@ class MediaRepository(object):
         self.dynamic_thumbnails = hs.config.dynamic_thumbnails
         self.thumbnail_requirements = hs.config.thumbnail_requirements
 
-        self.remote_media_linearizer = Linearizer()
+        self.remote_media_linearizer = Linearizer(name="media_remote")
 
         self.recently_accessed_remotes = set()
 

synapse/state.py

@@ -89,7 +89,7 @@ class StateHandler(object):
 
         # dict of set of event_ids -> _StateCacheEntry.
         self._state_cache = None
-        self.resolve_linearizer = Linearizer()
+        self.resolve_linearizer = Linearizer(name="state_resolve_lock")
 
     def start_caching(self):
         logger.debug("start_caching")

synapse/storage/__init__.py

@@ -189,7 +189,8 @@ class DataStore(RoomMemberStore, RoomStore,
             db_conn, "device_inbox",
             entity_column="user_id",
             stream_column="stream_id",
-            max_value=max_device_inbox_id
+            max_value=max_device_inbox_id,
+            limit=1000,
         )
         self._device_inbox_stream_cache = StreamChangeCache(
             "DeviceInboxStreamChangeCache", min_device_inbox_id,
@@ -202,6 +203,7 @@ class DataStore(RoomMemberStore, RoomStore,
             entity_column="destination",
             stream_column="stream_id",
             max_value=max_device_inbox_id,
+            limit=1000,
         )
         self._device_federation_outbox_stream_cache = StreamChangeCache(
             "DeviceFederationOutboxStreamChangeCache", min_device_outbox_id,

synapse/storage/_base.py

@@ -838,18 +838,19 @@ class SQLBaseStore(object):
         return txn.execute(sql, keyvalues.values())
 
     def _get_cache_dict(self, db_conn, table, entity_column, stream_column,
-                        max_value):
+                        max_value, limit=100000):
         # Fetch a mapping of room_id -> max stream position for "recent" rooms.
         # It doesn't really matter how many we get, the StreamChangeCache will
         # do the right thing to ensure it respects the max size of cache.
         sql = (
             "SELECT %(entity)s, MAX(%(stream)s) FROM %(table)s"
-            " WHERE %(stream)s > ? - 100000"
+            " WHERE %(stream)s > ? - %(limit)s"
             " GROUP BY %(entity)s"
         ) % {
             "table": table,
             "entity": entity_column,
             "stream": stream_column,
+            "limit": limit,
         }
 
         sql = self.database_engine.convert_param_style(sql)

synapse/storage/registration.py

@@ -413,6 +413,17 @@ class RegistrationStore(background_updates.BackgroundUpdateStore):
             desc="user_delete_threepids",
         )
 
+    def user_delete_threepid(self, user_id, medium, address):
+        return self._simple_delete(
+            "user_threepids",
+            keyvalues={
+                "user_id": user_id,
+                "medium": medium,
+                "address": address,
+            },
+            desc="user_delete_threepids",
+        )
+
     @defer.inlineCallbacks
     def count_all_users(self):
         """Counts all users registered on the homeserver."""

synapse/util/async.py

@@ -192,8 +192,11 @@ class Linearizer(object):
             logger.info(
                 "Waiting to acquire linearizer lock %r for key %r", self.name, key
             )
+            try:
                 with PreserveLoggingContext():
                     yield current_defer
+            except:
+                logger.exception("Unexpected exception in Linearizer")
 
         logger.info("Acquired linearizer lock %r for key %r", self.name, key)
 

tests/rest/client/v1/test_rooms.py

@@ -259,8 +259,8 @@ class RoomPermissionsTestCase(RestTestCase):
         # set [invite/join/left] of self, set [invite/join/left] of other,
         # expect all 404s because room doesn't exist on any server
         for usr in [self.user_id, self.rmcreator_id]:
-            yield self.join(room=room, user=usr, expect_code=403)
+            yield self.join(room=room, user=usr, expect_code=404)
-            yield self.leave(room=room, user=usr, expect_code=403)
+            yield self.leave(room=room, user=usr, expect_code=404)
 
     @defer.inlineCallbacks
     def test_membership_private_room_perms(self):

tests/rest/client/v1/utils.py

@@ -87,7 +87,10 @@ class RestTestCase(unittest.TestCase):
         (code, response) = yield self.mock_resource.trigger(
             "PUT", path, json.dumps(data)
         )
-        self.assertEquals(expect_code, code, msg=str(response))
+        self.assertEquals(
+            expect_code, code,
+            msg="Expected: %d, got: %d, resp: %r" % (expect_code, code, response)
+        )
 
         self.auth_user_id = temp_id