From dc60eee50e0a16ec24d10398c82f6fb64eec9ccc Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 20 Nov 2014 13:53:34 +0000 Subject: [PATCH] Refactor the HTTP clients a little. --- synapse/app/homeserver.py | 4 +- synapse/handlers/login.py | 4 +- synapse/handlers/register.py | 6 +- synapse/http/client.py | 154 +++++++++++++++++------------------ 4 files changed, 82 insertions(+), 86 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 85284a491..8f7d0cd74 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -26,7 +26,7 @@ from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect from synapse.http.content_repository import ContentRepoResource from synapse.http.server_key_resource import LocalKey -from synapse.http.client import MatrixHttpClient +from synapse.http.client import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, SERVER_KEY_PREFIX, @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) class SynapseHomeServer(HomeServer): def build_http_client(self): - return MatrixHttpClient(self) + return MatrixFederationHttpClient(self) def build_resource_for_client(self): return JsonResource() diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 99d15261d..1204dc3b8 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -17,7 +17,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import LoginError, Codes -from synapse.http.client import IdentityServerHttpClient +from synapse.http.client import SimpleHttpClient from synapse.util.emailutils import EmailException import synapse.util.emailutils as emailutils @@ -97,7 +97,7 @@ class LoginHandler(BaseHandler): @defer.inlineCallbacks def _query_email(self, email): - httpCli = IdentityServerHttpClient(self.hs) + httpCli = SimpleHttpClient(self.hs) data = yield httpCli.get_json( 'matrix.org:8090', # TODO FIXME This should be configurable. "/_matrix/identity/api/v1/lookup?medium=email&address=" + diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 7df9d9b82..a39230bc7 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -22,7 +22,7 @@ from synapse.api.errors import ( ) from ._base import BaseHandler import synapse.util.stringutils as stringutils -from synapse.http.client import IdentityServerHttpClient +from synapse.http.client import SimpleHttpClient from synapse.http.client import CaptchaServerHttpClient import base64 @@ -159,7 +159,7 @@ class RegistrationHandler(BaseHandler): def _threepid_from_creds(self, creds): # TODO: get this from the homeserver rather than creating a new one for # each request - httpCli = IdentityServerHttpClient(self.hs) + httpCli = SimpleHttpClient(self.hs) # XXX: make this configurable! trustedIdServers = ['matrix.org:8090'] if not creds['idServer'] in trustedIdServers: @@ -178,7 +178,7 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _bind_threepid(self, creds, mxid): - httpCli = IdentityServerHttpClient(self.hs) + httpCli = SimpleHttpClient(self.hs) data = yield httpCli.post_urlencoded_get_json( creds['idServer'], "/_matrix/identity/api/v1/3pid/bind", diff --git a/synapse/http/client.py b/synapse/http/client.py index e2fdded67..6361ac55f 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -154,16 +154,81 @@ class BaseHttpClient(object): defer.returnValue(response) -class MatrixHttpClient(BaseHttpClient): - """ Wrapper around the twisted HTTP client api. Implements +class SimpleHttpClient(BaseHttpClient): + """ + A simple, no-frills HTTP client with methods that wrap up common ways of using HTTP in Matrix + """ + def _getEndpoint(self, reactor, destination): + return matrix_endpoint(reactor, destination, timeout=10) + + @defer.inlineCallbacks + def post_urlencoded_get_json(self, destination, path, args={}): + logger.debug("post_urlencoded_get_json args: %s", args) + query_bytes = urllib.urlencode(args, True) + + def body_callback(method, url_bytes, headers_dict): + return FileBodyProducer(StringIO(query_bytes)) + + response = yield self._create_request( + destination.encode("ascii"), + "POST", + path.encode("ascii"), + body_callback=body_callback, + headers_dict={ + "Content-Type": ["application/x-www-form-urlencoded"] + } + ) + + body = yield readBody(response) + + defer.returnValue(json.loads(body)) + + @defer.inlineCallbacks + def get_json(self, destination, path, args={}, retry_on_dns_fail=True): + """ Get's some json from the given host and path + + Args: + destination (str): The remote server to send the HTTP request to. + path (str): The HTTP path. + args (dict): A dictionary used to create query strings, defaults to + None. + **Note**: The value of each key is assumed to be an iterable + and *not* a string. + + Returns: + Deferred: Succeeds when we get *any* HTTP response. + + The result of the deferred is a tuple of `(code, response)`, + where `response` is a dict representing the decoded JSON body. + """ + logger.debug("get_json args: %s", args) + + query_bytes = urllib.urlencode(args, True) + logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) + + response = yield self._create_request( + destination.encode("ascii"), + "GET", + path.encode("ascii"), + query_bytes=query_bytes, + retry_on_dns_fail=retry_on_dns_fail, + body_callback=None + ) + + body = yield readBody(response) + + defer.returnValue(json.loads(body)) + + +class MatrixFederationHttpClient(BaseHttpClient): + """HTTP client used to talk to other homeservers over the federation protocol. + Send client certificates and signs requests. Attributes: agent (twisted.web.client.Agent): The twisted Agent used to send the requests. """ - RETRY_DNS_LOOKUP_FAILURES = "__retry_dns" - def __init__(self, hs): self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname @@ -293,83 +358,17 @@ class MatrixHttpClient(BaseHttpClient): ) -class IdentityServerHttpClient(BaseHttpClient): - """Separate HTTP client for talking to the Identity servers since they - don't use SRV records and talk x-www-form-urlencoded rather than JSON. - """ - def _getEndpoint(self, reactor, destination): - #TODO: This should be talking TLS - return matrix_endpoint(reactor, destination, timeout=10) - - @defer.inlineCallbacks - def post_urlencoded_get_json(self, destination, path, args={}): - logger.debug("post_urlencoded_get_json args: %s", args) - query_bytes = urllib.urlencode(args, True) - - def body_callback(method, url_bytes, headers_dict): - return FileBodyProducer(StringIO(query_bytes)) - - response = yield self._create_request( - destination.encode("ascii"), - "POST", - path.encode("ascii"), - body_callback=body_callback, - headers_dict={ - "Content-Type": ["application/x-www-form-urlencoded"] - } - ) - - body = yield readBody(response) - - defer.returnValue(json.loads(body)) - - @defer.inlineCallbacks - def get_json(self, destination, path, args={}, retry_on_dns_fail=True): - """ Get's some json from the given host homeserver and path - - Args: - destination (str): The remote server to send the HTTP request - to. - path (str): The HTTP path. - args (dict): A dictionary used to create query strings, defaults to - None. - **Note**: The value of each key is assumed to be an iterable - and *not* a string. - - Returns: - Deferred: Succeeds when we get *any* HTTP response. - - The result of the deferred is a tuple of `(code, response)`, - where `response` is a dict representing the decoded JSON body. - """ - logger.debug("get_json args: %s", args) - - query_bytes = urllib.urlencode(args, True) - logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) - - response = yield self._create_request( - destination.encode("ascii"), - "GET", - path.encode("ascii"), - query_bytes=query_bytes, - retry_on_dns_fail=retry_on_dns_fail, - body_callback=None - ) - - body = yield readBody(response) - - defer.returnValue(json.loads(body)) - - class CaptchaServerHttpClient(BaseHttpClient): - """Separate HTTP client for talking to google's captcha servers""" + """ + Separate HTTP client for talking to google's captcha servers + Only slightly special because accepts partial download responses + """ def _getEndpoint(self, reactor, destination): return matrix_endpoint(reactor, destination, timeout=10) @defer.inlineCallbacks - def post_urlencoded_get_raw(self, destination, path, accept_partial=False, - args={}): + def post_urlencoded_get_raw(self, destination, path, args={}): query_bytes = urllib.urlencode(args, True) def body_callback(method, url_bytes, headers_dict): @@ -389,10 +388,7 @@ class CaptchaServerHttpClient(BaseHttpClient): body = yield readBody(response) defer.returnValue(body) except PartialDownloadError as e: - if accept_partial: - defer.returnValue(e.response) - else: - raise e + defer.returnValue(e.response) def _print_ex(e):