Split PlainHttpClient into separate clients for talking to Identity servers and talking to Capatcha servers

synapse/app/homeserver.py

@@ -25,7 +25,7 @@ from twisted.web.static import File
 from twisted.web.server import Site
 from synapse.http.server import JsonResource, RootRedirect
 from synapse.http.content_repository import ContentRepoResource
-from synapse.http.client import TwistedHttpClient
+from synapse.http.client import MatrixHttpClient
 from synapse.api.urls import (
     CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX
 )
@@ -47,7 +47,7 @@ logger = logging.getLogger(__name__)
 class SynapseHomeServer(HomeServer):
 
     def build_http_client(self):
-        return TwistedHttpClient(self)
+        return MatrixHttpClient(self)
 
     def build_resource_for_client(self):
         return JsonResource()

synapse/handlers/directory.py

@@ -18,7 +18,7 @@ from twisted.internet import defer
 from ._base import BaseHandler
 
 from synapse.api.errors import SynapseError
-from synapse.http.client import HttpClient
+from synapse.http.client import MatrixHttpClient
 from synapse.api.events.room import RoomAliasesEvent
 
 import logging
@@ -98,7 +98,7 @@ class DirectoryHandler(BaseHandler):
                 query_type="directory",
                 args={
                     "room_alias": room_alias.to_string(),
-                    HttpClient.RETRY_DNS_LOOKUP_FAILURES: False
+                    MatrixHttpClient.RETRY_DNS_LOOKUP_FAILURES: False
                 }
             )
 

synapse/handlers/login.py

@@ -17,7 +17,7 @@ from twisted.internet import defer
 
 from ._base import BaseHandler
 from synapse.api.errors import LoginError, Codes
-from synapse.http.client import PlainHttpClient
+from synapse.http.client import IdentityServerHttpClient
 from synapse.util.emailutils import EmailException
 import synapse.util.emailutils as emailutils
 
@@ -97,10 +97,10 @@ class LoginHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _query_email(self, email):
-        httpCli = PlainHttpClient(self.hs)
+        httpCli = IdentityServerHttpClient(self.hs)
         data = yield httpCli.get_json(
             'matrix.org:8090',  # TODO FIXME This should be configurable.
             "/_matrix/identity/api/v1/lookup?medium=email&address=" +
             "%s" % urllib.quote(email)
         )
-        defer.returnValue(data)
+        defer.returnValue(data)

synapse/handlers/register.py

@@ -22,7 +22,8 @@ from synapse.api.errors import (
 )
 from ._base import BaseHandler
 import synapse.util.stringutils as stringutils
-from synapse.http.client import PlainHttpClient
+from synapse.http.client import IdentityServerHttpClient
+from synapse.http.client import CaptchaServerHttpClient
 
 import base64
 import bcrypt
@@ -154,7 +155,9 @@ class RegistrationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _threepid_from_creds(self, creds):
-        httpCli = PlainHttpClient(self.hs)
+        # TODO: get this from the homeserver rather than creating a new one for
+        # each request
+        httpCli = IdentityServerHttpClient(self.hs)
         # XXX: make this configurable!
         trustedIdServers = ['matrix.org:8090']
         if not creds['idServer'] in trustedIdServers:
@@ -203,7 +206,9 @@ class RegistrationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _submit_captcha(self, ip_addr, private_key, challenge, response):
-        client = PlainHttpClient(self.hs)
+        # TODO: get this from the homeserver rather than creating a new one for
+        # each request
+        client = CaptchaServerHttpClient(self.hs)
         data = yield client.post_urlencoded_get_raw(
             "www.google.com:80",
             "/recaptcha/api/verify",

synapse/http/client.py

@@ -36,49 +36,6 @@ import urllib
 logger = logging.getLogger(__name__)
 
 
-class HttpClient(object):
-    """ Interface for talking json over http
-    """
-    RETRY_DNS_LOOKUP_FAILURES = "__retry_dns"
-
-    def put_json(self, destination, path, data):
-        """ Sends the specifed json data using PUT
-
-        Args:
-            destination (str): The remote server to send the HTTP request
-                to.
-            path (str): The HTTP path.
-            data (dict): A dict containing the data that will be used as
-                the request body. This will be encoded as JSON.
-
-        Returns:
-            Deferred: Succeeds when we get a 2xx HTTP response. The result
-            will be the decoded JSON body. On a 4xx or 5xx error response a
-            CodeMessageException is raised.
-        """
-        pass
-
-    def get_json(self, destination, path, args=None):
-        """ Get's some json from the given host homeserver and path
-
-        Args:
-            destination (str): The remote server to send the HTTP request
-                to.
-            path (str): The HTTP path.
-            args (dict): A dictionary used to create query strings, defaults to
-                None.
-                **Note**: The value of each key is assumed to be an iterable
-                and *not* a string.
-
-        Returns:
-            Deferred: Succeeds when we get *any* HTTP response.
-
-            The result of the deferred is a tuple of `(code, response)`,
-            where `response` is a dict representing the decoded JSON body.
-        """
-        pass
-
-
 class MatrixHttpAgent(_AgentBase):
 
     def __init__(self, reactor, pool=None):
@@ -102,113 +59,14 @@ class MatrixHttpAgent(_AgentBase):
                                          parsed_URI.originForm)
 
 
-class TwistedHttpClient(HttpClient):
+class BaseHttpClient(object):
-    """ Wrapper around the twisted HTTP client api.
+    """Base class for HTTP clients using twisted.
-
-    Attributes:
-        agent (twisted.web.client.Agent): The twisted Agent used to send the
-            requests.
     """
 
     def __init__(self, hs):
         self.agent = MatrixHttpAgent(reactor)
         self.hs = hs
 
-    @defer.inlineCallbacks
-    def put_json(self, destination, path, data, on_send_callback=None):
-        if destination in _destination_mappings:
-            destination = _destination_mappings[destination]
-
-        response = yield self._create_request(
-            destination.encode("ascii"),
-            "PUT",
-            path.encode("ascii"),
-            producer=_JsonProducer(data),
-            headers_dict={"Content-Type": ["application/json"]},
-            on_send_callback=on_send_callback,
-        )
-
-        logger.debug("Getting resp body")
-        body = yield readBody(response)
-        logger.debug("Got resp body")
-
-        defer.returnValue((response.code, body))
-
-    @defer.inlineCallbacks
-    def get_json(self, destination, path, args={}):
-        if destination in _destination_mappings:
-            destination = _destination_mappings[destination]
-
-        logger.debug("get_json args: %s", args)
-
-        retry_on_dns_fail = True
-        if HttpClient.RETRY_DNS_LOOKUP_FAILURES in args:
-            # FIXME: This isn't ideal, but the interface exposed in get_json
-            # isn't comprehensive enough to give caller's any control over
-            # their connection mechanics.
-            retry_on_dns_fail = args.pop(HttpClient.RETRY_DNS_LOOKUP_FAILURES)
-
-        query_bytes = urllib.urlencode(args, True)
-        logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)
-
-        response = yield self._create_request(
-            destination.encode("ascii"),
-            "GET",
-            path.encode("ascii"),
-            query_bytes=query_bytes,
-            retry_on_dns_fail=retry_on_dns_fail
-        )
-
-        body = yield readBody(response)
-
-        defer.returnValue(json.loads(body))
-
-    @defer.inlineCallbacks
-    def post_urlencoded_get_json(self, destination, path, args={}):
-        if destination in _destination_mappings:
-            destination = _destination_mappings[destination]
-
-        logger.debug("post_urlencoded_get_json args: %s", args)
-        query_bytes = urllib.urlencode(args, True)
-
-        response = yield self._create_request(
-            destination.encode("ascii"),
-            "POST",
-            path.encode("ascii"),
-            producer=FileBodyProducer(StringIO(urllib.urlencode(args))),
-            headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]}
-        )
-
-        body = yield readBody(response)
-
-        defer.returnValue(json.loads(body))
-        
-    # XXX FIXME : I'm so sorry.
-    @defer.inlineCallbacks
-    def post_urlencoded_get_raw(self, destination, path, accept_partial=False, args={}):
-        if destination in _destination_mappings:
-            destination = _destination_mappings[destination]
-
-        query_bytes = urllib.urlencode(args, True)
-
-        response = yield self._create_request(
-            destination.encode("ascii"),
-            "POST",
-            path.encode("ascii"),
-            producer=FileBodyProducer(StringIO(urllib.urlencode(args))),
-            headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]}
-        )
-
-        try:
-            body = yield readBody(response)
-            defer.returnValue(body)
-        except PartialDownloadError as e:
-            if accept_partial:
-                defer.returnValue(e.response)
-            else:
-                raise e
-        
-
     @defer.inlineCallbacks
     def _create_request(self, destination, method, path_bytes, param_bytes=b"",
                         query_bytes=b"", producer=None, headers_dict={},
@@ -232,7 +90,6 @@ class TwistedHttpClient(HttpClient):
 
         retries_left = 5
 
-        # TODO: setup and pass in an ssl_context to enable TLS
         endpoint = self._getEndpoint(reactor, destination);
 
         while True:
@@ -283,6 +140,92 @@ class TwistedHttpClient(HttpClient):
 
         defer.returnValue(response)
 
+
+class MatrixHttpClient(BaseHttpClient):
+    """ Wrapper around the twisted HTTP client api. Implements 
+
+    Attributes:
+        agent (twisted.web.client.Agent): The twisted Agent used to send the
+            requests.
+    """
+
+    RETRY_DNS_LOOKUP_FAILURES = "__retry_dns"
+
+    @defer.inlineCallbacks
+    def put_json(self, destination, path, data, on_send_callback=None):
+        """ Sends the specifed json data using PUT
+
+        Args:
+            destination (str): The remote server to send the HTTP request
+                to.
+            path (str): The HTTP path.
+            data (dict): A dict containing the data that will be used as
+                the request body. This will be encoded as JSON.
+
+        Returns:
+            Deferred: Succeeds when we get a 2xx HTTP response. The result
+            will be the decoded JSON body. On a 4xx or 5xx error response a
+            CodeMessageException is raised.
+        """
+        response = yield self._create_request(
+            destination.encode("ascii"),
+            "PUT",
+            path.encode("ascii"),
+            producer=_JsonProducer(data),
+            headers_dict={"Content-Type": ["application/json"]},
+            on_send_callback=on_send_callback,
+        )
+
+        logger.debug("Getting resp body")
+        body = yield readBody(response)
+        logger.debug("Got resp body")
+
+        defer.returnValue((response.code, body))
+
+    @defer.inlineCallbacks
+    def get_json(self, destination, path, args={}):
+        """ Get's some json from the given host homeserver and path
+
+        Args:
+            destination (str): The remote server to send the HTTP request
+                to.
+            path (str): The HTTP path.
+            args (dict): A dictionary used to create query strings, defaults to
+                None.
+                **Note**: The value of each key is assumed to be an iterable
+                and *not* a string.
+
+        Returns:
+            Deferred: Succeeds when we get *any* HTTP response.
+
+            The result of the deferred is a tuple of `(code, response)`,
+            where `response` is a dict representing the decoded JSON body.
+        """
+        logger.debug("get_json args: %s", args)
+
+        retry_on_dns_fail = True
+        if HttpClient.RETRY_DNS_LOOKUP_FAILURES in args:
+            # FIXME: This isn't ideal, but the interface exposed in get_json
+            # isn't comprehensive enough to give caller's any control over
+            # their connection mechanics.
+            retry_on_dns_fail = args.pop(HttpClient.RETRY_DNS_LOOKUP_FAILURES)
+
+        query_bytes = urllib.urlencode(args, True)
+        logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)
+
+        response = yield self._create_request(
+            destination.encode("ascii"),
+            "GET",
+            path.encode("ascii"),
+            query_bytes=query_bytes,
+            retry_on_dns_fail=retry_on_dns_fail
+        )
+
+        body = yield readBody(response)
+
+        defer.returnValue(json.loads(body))
+
+
     def _getEndpoint(self, reactor, destination):
         return matrix_endpoint(
             reactor, destination, timeout=10,
@@ -290,10 +233,69 @@ class TwistedHttpClient(HttpClient):
         )
 
 
-class PlainHttpClient(TwistedHttpClient):
+class IdentityServerHttpClient(BaseHttpClient):
+    """Separate HTTP client for talking to the Identity servers since they
+    don't use SRV records and talk x-www-form-urlencoded rather than JSON.
+    """
+    def _getEndpoint(self, reactor, destination):
+        #TODO: This should be talking TLS
+        return matrix_endpoint(reactor, destination, timeout=10)
+
+    @defer.inlineCallbacks
+    def post_urlencoded_get_json(self, destination, path, args={}):
+        if destination in _destination_mappings:
+            destination = _destination_mappings[destination]
+
+        logger.debug("post_urlencoded_get_json args: %s", args)
+        query_bytes = urllib.urlencode(args, True)
+
+        response = yield self._create_request(
+            destination.encode("ascii"),
+            "POST",
+            path.encode("ascii"),
+            producer=FileBodyProducer(StringIO(urllib.urlencode(args))),
+            headers_dict={
+                "Content-Type": ["application/x-www-form-urlencoded"]
+            }
+        )
+
+        body = yield readBody(response)
+
+        defer.returnValue(json.loads(body))
+
+
+class CaptchaServerHttpClient(MatrixHttpClient):
+    """Separate HTTP client for talking to google's captcha servers"""
+
     def _getEndpoint(self, reactor, destination):
         return matrix_endpoint(reactor, destination, timeout=10)
-    
+
+    @defer.inlineCallbacks
+    def post_urlencoded_get_raw(self, destination, path, accept_partial=False,
+                                args={}):
+        if destination in _destination_mappings:
+            destination = _destination_mappings[destination]
+
+        query_bytes = urllib.urlencode(args, True)
+
+        response = yield self._create_request(
+            destination.encode("ascii"),
+            "POST",
+            path.encode("ascii"),
+            producer=FileBodyProducer(StringIO(urllib.urlencode(args))),
+            headers_dict={
+                "Content-Type": ["application/x-www-form-urlencoded"]
+            }
+        )
+
+        try:
+            body = yield readBody(response)
+            defer.returnValue(body)
+        except PartialDownloadError as e:
+            if accept_partial:
+                defer.returnValue(e.response)
+            else:
+                raise e
 
 def _print_ex(e):
     if hasattr(e, "reasons") and e.reasons:

tests/handlers/test_directory.py

@@ -20,7 +20,7 @@ from twisted.internet import defer
 from mock import Mock
 
 from synapse.server import HomeServer
-from synapse.http.client import HttpClient
+from synapse.http.client import MatrixHttpClient
 from synapse.handlers.directory import DirectoryHandler
 from synapse.storage.directory import RoomAliasMapping
 
@@ -95,7 +95,7 @@ class DirectoryTestCase(unittest.TestCase):
             query_type="directory",
             args={
                 "room_alias": "#another:remote",
-                HttpClient.RETRY_DNS_LOOKUP_FAILURES: False
+                MatrixHttpClient.RETRY_DNS_LOOKUP_FAILURES: False
             }
         )