Merge branch 'http_client_refactor' into pushers

CHANGES.rst

@@ -1,3 +1,27 @@
+Changes in synapse 0.5.0 (2014-11-19)
+=====================================
+This release includes changes to the federation protocol and client-server API
+that is not backwards compatible.
+
+This release also changes the internal database schemas and so requires servers to
+drop their current history. See UPGRADES.rst for details.
+
+Homeserver:
+ * Add authentication and authorization to the federation protocol. Events are
+   now signed by their originating homeservers.
+ * Implement the new authorization model for rooms.
+ * Split out web client into a seperate repository: matrix-angular-sdk.
+ * Change the structure of PDUs.
+ * Fix bug where user could not join rooms via an alias containing 4-byte
+   UTF-8 characters.
+ * Merge concept of PDUs and Events internally.
+ * Improve logging by adding request ids to log lines.
+ * Implement a very basic room initial sync API.
+ * Implement the new invite/join federation APIs.
+
+Webclient:
+ * The webclient has been moved to a seperate repository.
+
 Changes in synapse 0.4.2 (2014-10-31)
 =====================================
 

UPGRADE.rst

@@ -1,3 +1,39 @@
+Upgrading to v0.5.0
+===================
+
+The webclient has been split out into a seperate repository/pacakage in this
+release. Before you restart your homeserver you will need to pull in the
+webclient package by running::
+
+  python setup.py develop --user
+
+This release completely changes the database schema and so requires upgrading
+it before starting the new version of the homeserver.
+
+The script "database-prepare-for-0.5.0.sh" should be used to upgrade the
+database. This will save all user information, such as logins and profiles, 
+but will otherwise purge the database. This includes messages, which
+rooms the home server was a member of and room alias mappings.
+
+If you would like to keep your history, please take a copy of your database
+file and ask for help in #matrix:matrix.org. The upgrade process is,
+unfortunately, non trivial and requires human intervention to resolve any
+resulting conflicts during the upgrade process.
+
+Before running the command the homeserver should be first completely 
+shutdown. To run it, simply specify the location of the database, e.g.:
+
+  ./database-prepare-for-0.5.0.sh "homeserver.db"
+
+Once this has successfully completed it will be safe to restart the 
+homeserver. You may notice that the homeserver takes a few seconds longer to 
+restart than usual as it reinitializes the database.
+
+On startup of the new version, users can either rejoin remote rooms using room
+aliases or by being reinvited. Alternatively, if any other homeserver sends a
+message to a room that the homeserver was previously in the local HS will 
+automatically rejoin the room.
+
 Upgrading to v0.4.0
 ===================
 

VERSION

@@ -1 +1 @@
-0.4.2
+0.5.0

database-prepare-for-0.5.0.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# This is will prepare a synapse database for running with v0.5.0 of synapse. 
+# It will store all the user information, but will *delete* all messages and
+# room data.
+
+set -e
+
+cp "$1" "$1.bak"
+
+DUMP=$(sqlite3 "$1" << 'EOF'
+.dump users
+.dump access_tokens
+.dump presence
+.dump profiles
+EOF
+)
+
+rm "$1"
+
+sqlite3 "$1" <<< "$DUMP" 

setup.py

@@ -26,13 +26,13 @@ def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read()
 
 setup(
-    name="synapse",
-    version=read("VERSION"),
+    name="matrix-synapse",
+    version=read("VERSION").strip(),
     packages=find_packages(exclude=["tests", "tests.*"]),
     description="Reference Synapse Home Server",
     install_requires=[
         "syutil==0.0.2",
-        "syweb==0.0.1",
+        "matrix_angular_sdk==0.5.0",
         "Twisted>=14.0.0",
         "service_identity>=1.0.0",
         "pyopenssl>=0.14",
@@ -45,7 +45,7 @@ setup(
     dependency_links=[
         "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2",
         "https://github.com/pyca/pynacl/tarball/52dbe2dc33f1#egg=pynacl-0.3.0",
-        "https://github.com/matrix-org/matrix-angular-sdk/tarball/master/#egg=syweb-0.0.1",
+        "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.5.0/#egg=matrix_angular_sdk-0.5.0",
     ],
     setup_requires=[
         "setuptools_trial",

synapse/__init__.py

@@ -16,4 +16,4 @@
 """ This is a reference implementation of a synapse home server.
 """
 
-__version__ = "0.4.2"
+__version__ = "0.5.0"

synapse/app/homeserver.py

@@ -26,7 +26,7 @@ from twisted.web.server import Site
 from synapse.http.server import JsonResource, RootRedirect
 from synapse.http.content_repository import ContentRepoResource
 from synapse.http.server_key_resource import LocalKey
-from synapse.http.client import MatrixHttpClient
+from synapse.http.client import MatrixFederationHttpClient
 from synapse.api.urls import (
     CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX,
     SERVER_KEY_PREFIX,
@@ -51,7 +51,7 @@ logger = logging.getLogger(__name__)
 class SynapseHomeServer(HomeServer):
 
     def build_http_client(self):
-        return MatrixHttpClient(self)
+        return MatrixFederationHttpClient(self)
 
     def build_resource_for_client(self):
         return JsonResource()

synapse/handlers/directory.py

@@ -128,7 +128,7 @@ class DirectoryHandler(BaseHandler):
                 "servers": result.servers,
             })
         else:
-            raise SynapseError(404, "Room alias \"%s\" not found", room_alias)
+            raise SynapseError(404, "Room alias \"%s\" not found" % (room_alias,))
 
 
     @defer.inlineCallbacks

synapse/handlers/login.py

@@ -17,7 +17,7 @@ from twisted.internet import defer
 
 from ._base import BaseHandler
 from synapse.api.errors import LoginError, Codes
-from synapse.http.client import IdentityServerHttpClient
+from synapse.http.client import SimpleHttpClient
 from synapse.util.emailutils import EmailException
 import synapse.util.emailutils as emailutils
 
@@ -97,7 +97,7 @@ class LoginHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _query_email(self, email):
-        httpCli = IdentityServerHttpClient(self.hs)
+        httpCli = SimpleHttpClient(self.hs)
         data = yield httpCli.get_json(
             'matrix.org:8090',  # TODO FIXME This should be configurable.
             "/_matrix/identity/api/v1/lookup?medium=email&address=" +

synapse/handlers/register.py

@@ -22,7 +22,7 @@ from synapse.api.errors import (
 )
 from ._base import BaseHandler
 import synapse.util.stringutils as stringutils
-from synapse.http.client import IdentityServerHttpClient
+from synapse.http.client import SimpleHttpClient
 from synapse.http.client import CaptchaServerHttpClient
 
 import base64
@@ -159,7 +159,7 @@ class RegistrationHandler(BaseHandler):
     def _threepid_from_creds(self, creds):
         # TODO: get this from the homeserver rather than creating a new one for
         # each request
-        httpCli = IdentityServerHttpClient(self.hs)
+        httpCli = SimpleHttpClient(self.hs)
         # XXX: make this configurable!
         trustedIdServers = ['matrix.org:8090']
         if not creds['idServer'] in trustedIdServers:
@@ -178,7 +178,7 @@ class RegistrationHandler(BaseHandler):
 
     @defer.inlineCallbacks
     def _bind_threepid(self, creds, mxid):
-        httpCli = IdentityServerHttpClient(self.hs)
+        httpCli = SimpleHttpClient(self.hs)
         data = yield httpCli.post_urlencoded_get_json(
             creds['idServer'],
             "/_matrix/identity/api/v1/3pid/bind",

synapse/http/client.py

@@ -154,16 +154,81 @@ class BaseHttpClient(object):
         defer.returnValue(response)
 
 
-class MatrixHttpClient(BaseHttpClient):
-    """ Wrapper around the twisted HTTP client api. Implements
+class SimpleHttpClient(BaseHttpClient):
+    """
+    A simple, no-frills HTTP client with methods that wrap up common ways of using HTTP in Matrix
+    """
+    def _getEndpoint(self, reactor, destination):
+        return matrix_endpoint(reactor, destination, timeout=10)
+
+    @defer.inlineCallbacks
+    def post_urlencoded_get_json(self, destination, path, args={}):
+        logger.debug("post_urlencoded_get_json args: %s", args)
+        query_bytes = urllib.urlencode(args, True)
+
+        def body_callback(method, url_bytes, headers_dict):
+            return FileBodyProducer(StringIO(query_bytes))
+
+        response = yield self._create_request(
+            destination.encode("ascii"),
+            "POST",
+            path.encode("ascii"),
+            body_callback=body_callback,
+            headers_dict={
+                "Content-Type": ["application/x-www-form-urlencoded"]
+            }
+        )
+
+        body = yield readBody(response)
+
+        defer.returnValue(json.loads(body))
+
+    @defer.inlineCallbacks
+    def get_json(self, destination, path, args={}, retry_on_dns_fail=True):
+        """ Get's some json from the given host and path
+
+        Args:
+            destination (str): The remote server to send the HTTP request to.
+            path (str): The HTTP path.
+            args (dict): A dictionary used to create query strings, defaults to
+                None.
+                **Note**: The value of each key is assumed to be an iterable
+                and *not* a string.
+
+        Returns:
+            Deferred: Succeeds when we get *any* HTTP response.
+
+            The result of the deferred is a tuple of `(code, response)`,
+            where `response` is a dict representing the decoded JSON body.
+        """
+        logger.debug("get_json args: %s", args)
+
+        query_bytes = urllib.urlencode(args, True)
+        logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)
+
+        response = yield self._create_request(
+            destination.encode("ascii"),
+            "GET",
+            path.encode("ascii"),
+            query_bytes=query_bytes,
+            retry_on_dns_fail=retry_on_dns_fail,
+            body_callback=None
+        )
+
+        body = yield readBody(response)
+
+        defer.returnValue(json.loads(body))
+
+
+class MatrixFederationHttpClient(BaseHttpClient):
+    """HTTP client used to talk to other homeservers over the federation protocol.
+    Send client certificates and signs requests.
 
     Attributes:
         agent (twisted.web.client.Agent): The twisted Agent used to send the
             requests.
     """
 
-    RETRY_DNS_LOOKUP_FAILURES = "__retry_dns"
-
     def __init__(self, hs):
         self.signing_key = hs.config.signing_key[0]
         self.server_name = hs.hostname
@@ -293,83 +358,17 @@ class MatrixHttpClient(BaseHttpClient):
         )
 
 
-class IdentityServerHttpClient(BaseHttpClient):
-    """Separate HTTP client for talking to the Identity servers since they
-    don't use SRV records and talk x-www-form-urlencoded rather than JSON.
+class CaptchaServerHttpClient(BaseHttpClient):
     """
-    def _getEndpoint(self, reactor, destination):
-        #TODO: This should be talking TLS
-        return matrix_endpoint(reactor, destination, timeout=10)
-
-    @defer.inlineCallbacks
-    def post_urlencoded_get_json(self, destination, path, args={}):
-        logger.debug("post_urlencoded_get_json args: %s", args)
-        query_bytes = urllib.urlencode(args, True)
-
-        def body_callback(method, url_bytes, headers_dict):
-            return FileBodyProducer(StringIO(query_bytes))
-
-        response = yield self._create_request(
-            destination.encode("ascii"),
-            "POST",
-            path.encode("ascii"),
-            body_callback=body_callback,
-            headers_dict={
-                "Content-Type": ["application/x-www-form-urlencoded"]
-            }
-        )
-
-        body = yield readBody(response)
-
-        defer.returnValue(json.loads(body))
-
-    @defer.inlineCallbacks
-    def get_json(self, destination, path, args={}, retry_on_dns_fail=True):
-        """ Get's some json from the given host homeserver and path
-
-        Args:
-            destination (str): The remote server to send the HTTP request
-                to.
-            path (str): The HTTP path.
-            args (dict): A dictionary used to create query strings, defaults to
-                None.
-                **Note**: The value of each key is assumed to be an iterable
-                and *not* a string.
-
-        Returns:
-            Deferred: Succeeds when we get *any* HTTP response.
-
-            The result of the deferred is a tuple of `(code, response)`,
-            where `response` is a dict representing the decoded JSON body.
+    Separate HTTP client for talking to google's captcha servers
+    Only slightly special because accepts partial download responses
     """
-        logger.debug("get_json args: %s", args)
-
-        query_bytes = urllib.urlencode(args, True)
-        logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)
-
-        response = yield self._create_request(
-            destination.encode("ascii"),
-            "GET",
-            path.encode("ascii"),
-            query_bytes=query_bytes,
-            retry_on_dns_fail=retry_on_dns_fail,
-            body_callback=None
-        )
-
-        body = yield readBody(response)
-
-        defer.returnValue(json.loads(body))
-
-
-class CaptchaServerHttpClient(MatrixHttpClient):
-    """Separate HTTP client for talking to google's captcha servers"""
 
     def _getEndpoint(self, reactor, destination):
         return matrix_endpoint(reactor, destination, timeout=10)
 
     @defer.inlineCallbacks
-    def post_urlencoded_get_raw(self, destination, path, accept_partial=False,
-                                args={}):
+    def post_urlencoded_get_raw(self, destination, path, args={}):
         query_bytes = urllib.urlencode(args, True)
 
         def body_callback(method, url_bytes, headers_dict):
@@ -389,10 +388,7 @@ class CaptchaServerHttpClient(MatrixHttpClient):
             body = yield readBody(response)
             defer.returnValue(body)
         except PartialDownloadError as e:
-            if accept_partial:
             defer.returnValue(e.response)
-            else:
-                raise e
 
 
 def _print_ex(e):