Refactor the HTTP clients a little.

synapse/app/homeserver.py

@@ -26,7 +26,7 @@ from twisted.web.server import Site
 from synapse.http.server import JsonResource, RootRedirect
 from synapse.http.content_repository import ContentRepoResource
 from synapse.http.server_key_resource import LocalKey
-from synapse.http.client import MatrixHttpClient
+from synapse.http.client import MatrixFederationHttpClient
 from synapse.api.urls import (
     CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX,
     SERVER_KEY_PREFIX,
@@ -51,7 +51,7 @@ logger = logging.getLogger(__name__)
 class SynapseHomeServer(HomeServer):
 
     def build_http_client(self):
-        return MatrixHttpClient(self)
+        return MatrixFederationHttpClient(self)
 
     def build_resource_for_client(self):
         return JsonResource()

synapse/handlers/login.py

@@ -17,7 +17,7 @@ from twisted.internet import defer
 
 from ._base import BaseHandler
 from synapse.api.errors import LoginError, Codes
-from synapse.http.client import IdentityServerHttpClient
+from synapse.http.client import SimpleHttpClient
 from synapse.util.emailutils import EmailException
 import synapse.util.emailutils as emailutils
 
@@ -97,7 +97,7 @@ class LoginHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _query_email(self, email):
-        httpCli = IdentityServerHttpClient(self.hs)
+        httpCli = SimpleHttpClient(self.hs)
         data = yield httpCli.get_json(
             'matrix.org:8090',  # TODO FIXME This should be configurable.
             "/_matrix/identity/api/v1/lookup?medium=email&address=" +

synapse/handlers/register.py

@@ -22,7 +22,7 @@ from synapse.api.errors import (
 )
 from ._base import BaseHandler
 import synapse.util.stringutils as stringutils
-from synapse.http.client import IdentityServerHttpClient
+from synapse.http.client import SimpleHttpClient
 from synapse.http.client import CaptchaServerHttpClient
 
 import base64
@@ -159,7 +159,7 @@ class RegistrationHandler(BaseHandler):
     def _threepid_from_creds(self, creds):
         # TODO: get this from the homeserver rather than creating a new one for
         # each request
-        httpCli = IdentityServerHttpClient(self.hs)
+        httpCli = SimpleHttpClient(self.hs)
         # XXX: make this configurable!
         trustedIdServers = ['matrix.org:8090']
         if not creds['idServer'] in trustedIdServers:
@@ -178,7 +178,7 @@ class RegistrationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _bind_threepid(self, creds, mxid):
-        httpCli = IdentityServerHttpClient(self.hs)
+        httpCli = SimpleHttpClient(self.hs)
         data = yield httpCli.post_urlencoded_get_json(
             creds['idServer'],
             "/_matrix/identity/api/v1/3pid/bind",

synapse/http/client.py

@@ -154,16 +154,81 @@ class BaseHttpClient(object):
         defer.returnValue(response)
 
 
-class MatrixHttpClient(BaseHttpClient):
+class SimpleHttpClient(BaseHttpClient):
-    """ Wrapper around the twisted HTTP client api. Implements
+    """
+    A simple, no-frills HTTP client with methods that wrap up common ways of using HTTP in Matrix
+    """
+    def _getEndpoint(self, reactor, destination):
+        return matrix_endpoint(reactor, destination, timeout=10)
+
+    @defer.inlineCallbacks
+    def post_urlencoded_get_json(self, destination, path, args={}):
+        logger.debug("post_urlencoded_get_json args: %s", args)
+        query_bytes = urllib.urlencode(args, True)
+
+        def body_callback(method, url_bytes, headers_dict):
+            return FileBodyProducer(StringIO(query_bytes))
+
+        response = yield self._create_request(
+            destination.encode("ascii"),
+            "POST",
+            path.encode("ascii"),
+            body_callback=body_callback,
+            headers_dict={
+                "Content-Type": ["application/x-www-form-urlencoded"]
+            }
+        )
+
+        body = yield readBody(response)
+
+        defer.returnValue(json.loads(body))
+
+    @defer.inlineCallbacks
+    def get_json(self, destination, path, args={}, retry_on_dns_fail=True):
+        """ Get's some json from the given host and path
+
+        Args:
+            destination (str): The remote server to send the HTTP request to.
+            path (str): The HTTP path.
+            args (dict): A dictionary used to create query strings, defaults to
+                None.
+                **Note**: The value of each key is assumed to be an iterable
+                and *not* a string.
+
+        Returns:
+            Deferred: Succeeds when we get *any* HTTP response.
+
+            The result of the deferred is a tuple of `(code, response)`,
+            where `response` is a dict representing the decoded JSON body.
+        """
+        logger.debug("get_json args: %s", args)
+
+        query_bytes = urllib.urlencode(args, True)
+        logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)
+
+        response = yield self._create_request(
+            destination.encode("ascii"),
+            "GET",
+            path.encode("ascii"),
+            query_bytes=query_bytes,
+            retry_on_dns_fail=retry_on_dns_fail,
+            body_callback=None
+        )
+
+        body = yield readBody(response)
+
+        defer.returnValue(json.loads(body))
+
+
+class MatrixFederationHttpClient(BaseHttpClient):
+    """HTTP client used to talk to other homeservers over the federation protocol.
+    Send client certificates and signs requests.
 
     Attributes:
         agent (twisted.web.client.Agent): The twisted Agent used to send the
             requests.
     """
 
-    RETRY_DNS_LOOKUP_FAILURES = "__retry_dns"
-
     def __init__(self, hs):
         self.signing_key = hs.config.signing_key[0]
         self.server_name = hs.hostname
@@ -293,83 +358,17 @@ class MatrixHttpClient(BaseHttpClient):
         )
 
 
-class IdentityServerHttpClient(BaseHttpClient):
-    """Separate HTTP client for talking to the Identity servers since they
-    don't use SRV records and talk x-www-form-urlencoded rather than JSON.
-    """
-    def _getEndpoint(self, reactor, destination):
-        #TODO: This should be talking TLS
-        return matrix_endpoint(reactor, destination, timeout=10)
-
-    @defer.inlineCallbacks
-    def post_urlencoded_get_json(self, destination, path, args={}):
-        logger.debug("post_urlencoded_get_json args: %s", args)
-        query_bytes = urllib.urlencode(args, True)
-
-        def body_callback(method, url_bytes, headers_dict):
-            return FileBodyProducer(StringIO(query_bytes))
-
-        response = yield self._create_request(
-            destination.encode("ascii"),
-            "POST",
-            path.encode("ascii"),
-            body_callback=body_callback,
-            headers_dict={
-                "Content-Type": ["application/x-www-form-urlencoded"]
-            }
-        )
-
-        body = yield readBody(response)
-
-        defer.returnValue(json.loads(body))
-
-    @defer.inlineCallbacks
-    def get_json(self, destination, path, args={}, retry_on_dns_fail=True):
-        """ Get's some json from the given host homeserver and path
-
-        Args:
-            destination (str): The remote server to send the HTTP request
-                to.
-            path (str): The HTTP path.
-            args (dict): A dictionary used to create query strings, defaults to
-                None.
-                **Note**: The value of each key is assumed to be an iterable
-                and *not* a string.
-
-        Returns:
-            Deferred: Succeeds when we get *any* HTTP response.
-
-            The result of the deferred is a tuple of `(code, response)`,
-            where `response` is a dict representing the decoded JSON body.
-        """
-        logger.debug("get_json args: %s", args)
-
-        query_bytes = urllib.urlencode(args, True)
-        logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)
-
-        response = yield self._create_request(
-            destination.encode("ascii"),
-            "GET",
-            path.encode("ascii"),
-            query_bytes=query_bytes,
-            retry_on_dns_fail=retry_on_dns_fail,
-            body_callback=None
-        )
-
-        body = yield readBody(response)
-
-        defer.returnValue(json.loads(body))
-
-
 class CaptchaServerHttpClient(BaseHttpClient):
-    """Separate HTTP client for talking to google's captcha servers"""
+    """
+    Separate HTTP client for talking to google's captcha servers
+    Only slightly special because accepts partial download responses
+    """
 
     def _getEndpoint(self, reactor, destination):
         return matrix_endpoint(reactor, destination, timeout=10)
 
     @defer.inlineCallbacks
-    def post_urlencoded_get_raw(self, destination, path, accept_partial=False,
+    def post_urlencoded_get_raw(self, destination, path, args={}):
-                                args={}):
         query_bytes = urllib.urlencode(args, True)
 
         def body_callback(method, url_bytes, headers_dict):
@@ -389,10 +388,7 @@ class CaptchaServerHttpClient(BaseHttpClient):
             body = yield readBody(response)
             defer.returnValue(body)
         except PartialDownloadError as e:
-            if accept_partial:
+            defer.returnValue(e.response)
-                defer.returnValue(e.response)
-            else:
-                raise e
 
 
 def _print_ex(e):