From 90abdaf3bcd3eceb6dd5688f9ef3623382883173 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 1 Jun 2015 10:51:50 +0100
Subject: [PATCH 01/38] Use Twisted-15.2.1, Use Agent.usingEndpointFactory
 rather than implement our own Agent

---
 synapse/http/matrixfederationclient.py | 75 +++++++++-----------------
 synapse/python_dependencies.py         |  2 +-
 2 files changed, 26 insertions(+), 51 deletions(-)

diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 7f3d8fc88..ec5b06ddc 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -16,7 +16,7 @@
 
 from twisted.internet import defer, reactor, protocol
 from twisted.internet.error import DNSLookupError
-from twisted.web.client import readBody, _AgentBase, _URI, HTTPConnectionPool
+from twisted.web.client import readBody, HTTPConnectionPool, Agent
 from twisted.web.http_headers import Headers
 from twisted.web._newclient import ResponseDone
 
@@ -53,41 +53,17 @@ incoming_responses_counter = metrics.register_counter(
 )
 
 
-class MatrixFederationHttpAgent(_AgentBase):
+class MatrixFederationEndpointFactory(object):
+    def __init__(self, hs):
+        self.tls_context_factory = hs.tls_context_factory
 
-    def __init__(self, reactor, pool=None):
-        _AgentBase.__init__(self, reactor, pool)
+    def endpointForURI(self, uri):
+        destination = uri.netloc
 
-    def request(self, destination, endpoint, method, path, params, query,
-                headers, body_producer):
-
-        outgoing_requests_counter.inc(method)
-
-        host = b""
-        port = 0
-        fragment = b""
-
-        parsed_URI = _URI(b"http", destination, host, port, path, params,
-                          query, fragment)
-
-        # Set the connection pool key to be the destination.
-        key = destination
-
-        d = self._requestWithEndpoint(key, endpoint, method, parsed_URI,
-                                      headers, body_producer,
-                                      parsed_URI.originForm)
-
-        def _cb(response):
-            incoming_responses_counter.inc(method, response.code)
-            return response
-
-        def _eb(failure):
-            incoming_responses_counter.inc(method, "ERR")
-            return failure
-
-        d.addCallbacks(_cb, _eb)
-
-        return d
+        return matrix_federation_endpoint(
+            reactor, destination, timeout=10,
+            ssl_context_factory=self.tls_context_factory
+        )
 
 
 class MatrixFederationHttpClient(object):
@@ -105,10 +81,17 @@ class MatrixFederationHttpClient(object):
         self.server_name = hs.hostname
         pool = HTTPConnectionPool(reactor)
         pool.maxPersistentPerHost = 10
-        self.agent = MatrixFederationHttpAgent(reactor, pool=pool)
+        self.agent = Agent.usingEndpointFactory(
+            reactor, MatrixFederationEndpointFactory(hs), pool=pool
+        )
         self.clock = hs.get_clock()
         self.version_string = hs.version_string
 
+    def _create_url(self, destination, path_bytes, param_bytes, query_bytes):
+        return urlparse.urlunparse(
+            ("matrix", destination, path_bytes, param_bytes, query_bytes, "")
+        )
+
     @defer.inlineCallbacks
     def _create_request(self, destination, method, path_bytes,
                         body_callback, headers_dict={}, param_bytes=b"",
@@ -119,8 +102,8 @@ class MatrixFederationHttpClient(object):
         headers_dict[b"User-Agent"] = [self.version_string]
         headers_dict[b"Host"] = [destination]
 
-        url_bytes = urlparse.urlunparse(
-            ("", "", path_bytes, param_bytes, query_bytes, "",)
+        url_bytes = self._create_url(
+            destination, path_bytes, param_bytes, query_bytes
         )
 
         logger.info("Sending request to %s: %s %s",
@@ -139,22 +122,20 @@ class MatrixFederationHttpClient(object):
         # (once we have reliable transactions in place)
         retries_left = 5
 
-        endpoint = self._getEndpoint(reactor, destination)
+        http_url_bytes = urlparse.urlunparse(
+            ("", "", path_bytes, param_bytes, query_bytes, "")
+        )
 
         while True:
             producer = None
             if body_callback:
-                producer = body_callback(method, url_bytes, headers_dict)
+                producer = body_callback(method, http_url_bytes, headers_dict)
 
             try:
                 request_deferred = preserve_context_over_fn(
                     self.agent.request,
-                    destination,
-                    endpoint,
                     method,
-                    path_bytes,
-                    param_bytes,
-                    query_bytes,
+                    url_bytes,
                     Headers(headers_dict),
                     producer
                 )
@@ -442,12 +423,6 @@ class MatrixFederationHttpClient(object):
 
         defer.returnValue((length, headers))
 
-    def _getEndpoint(self, reactor, destination):
-        return matrix_federation_endpoint(
-            reactor, destination, timeout=10,
-            ssl_context_factory=self.hs.tls_context_factory
-        )
-
 
 class _ReadBodyToFileProtocol(protocol.Protocol):
     def __init__(self, stream, deferred, max_size):
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index a45dd3c93..d1bf6d50c 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
 
 REQUIREMENTS = {
     "syutil>=0.0.6": ["syutil>=0.0.6"],
-    "Twisted==14.0.2": ["twisted==14.0.2"],
+    "Twisted==15.2.1": ["twisted==15.2.1"],
     "service_identity>=1.0.0": ["service_identity>=1.0.0"],
     "pyopenssl>=0.14": ["OpenSSL>=0.14"],
     "pyyaml": ["yaml"],

From 378a0f7a79a22a1211e3d048b7fd9e99d1efc7de Mon Sep 17 00:00:00 2001
From: manuroe <manu@matrix.org>
Date: Thu, 4 Jun 2015 17:58:17 +0200
Subject: [PATCH 02/38] Federation demo start.sh: Fixed --no-rate-limit param
 in the script

---
 demo/clean.sh |  6 ++++--
 demo/start.sh | 17 +++++++++--------
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/demo/clean.sh b/demo/clean.sh
index c5dabd476..31007737f 100755
--- a/demo/clean.sh
+++ b/demo/clean.sh
@@ -11,7 +11,9 @@ if [ -f $PID_FILE ]; then
     exit 1
 fi
 
-find "$DIR" -name "*.log" -delete
-find "$DIR" -name "*.db" -delete
+for port in 8080 8081 8082; do
+    rm -rf demo/$port
+    rm -rf demo/media_store.$port
+done
 
 rm -rf $DIR/etc
diff --git a/demo/start.sh b/demo/start.sh
index b9cc14b9d..b5dea5e17 100755
--- a/demo/start.sh
+++ b/demo/start.sh
@@ -8,14 +8,6 @@ cd "$DIR/.."
 
 mkdir -p demo/etc
 
-# Check the --no-rate-limit param
-PARAMS=""
-if [ $# -eq 1 ]; then
-    if [ $1 = "--no-rate-limit" ]; then
-	    PARAMS="--rc-messages-per-second 1000 --rc-message-burst-count 1000"
-    fi
-fi
-
 export PYTHONPATH=$(readlink -f $(pwd))
 
 
@@ -35,6 +27,15 @@ for port in 8080 8081 8082; do
         -H "localhost:$https_port" \
         --config-path "$DIR/etc/$port.config" \
 
+    # Check script parameters
+    if [ $# -eq 1 ]; then
+        if [ $1 = "--no-rate-limit" ]; then
+            # Set high limits in config file to disable rate limiting
+            perl -p -i -e 's/rc_messages_per_second.*/rc_messages_per_second: 1000/g' $DIR/etc/$port.config
+            perl -p -i -e 's/rc_message_burst_count.*/rc_message_burst_count: 1000/g' $DIR/etc/$port.config
+        fi
+    fi
+
     python -m synapse.app.homeserver \
         --config-path "$DIR/etc/$port.config" \
         -D \

From 66da8f60d057527fe74ff5790973d2865c980efb Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 15 Jun 2015 16:27:20 +0100
Subject: [PATCH 03/38] Bump the version of twisted needed for setup_requires
 to 15.2.1

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index f9929591e..c790484c7 100755
--- a/setup.py
+++ b/setup.py
@@ -48,7 +48,7 @@ setup(
     description="Reference Synapse Home Server",
     install_requires=dependencies['requirements'](include_conditional=True).keys(),
     setup_requires=[
-        "Twisted==14.0.2", # Here to override setuptools_trial's dependency on Twisted>=2.4.0
+        "Twisted==15.2.1", # Here to override setuptools_trial's dependency on Twisted>=2.4.0
         "setuptools_trial",
         "mock"
     ],

From 2b4f47db9c13840c4b9dbbffb28d7860fe007d68 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 23 Jul 2015 14:52:29 +0100
Subject: [PATCH 04/38] Generate local thumbnails on a thread

---
 synapse/rest/media/v1/base_resource.py | 77 ++++++++++++++------------
 1 file changed, 43 insertions(+), 34 deletions(-)

diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py
index c43ae0314..84e1961a2 100644
--- a/synapse/rest/media/v1/base_resource.py
+++ b/synapse/rest/media/v1/base_resource.py
@@ -244,43 +244,52 @@ class BaseMediaResource(Resource):
             )
             return
 
-        scales = set()
-        crops = set()
-        for r_width, r_height, r_method, r_type in requirements:
-            if r_method == "scale":
-                t_width, t_height = thumbnailer.aspect(r_width, r_height)
-                scales.add((
-                    min(m_width, t_width), min(m_height, t_height), r_type,
+        local_thumbnails = []
+
+        def generate_thumbnails():
+            scales = set()
+            crops = set()
+            for r_width, r_height, r_method, r_type in requirements:
+                if r_method == "scale":
+                    t_width, t_height = thumbnailer.aspect(r_width, r_height)
+                    scales.add((
+                        min(m_width, t_width), min(m_height, t_height), r_type,
+                    ))
+                elif r_method == "crop":
+                    crops.add((r_width, r_height, r_type))
+
+            for t_width, t_height, t_type in scales:
+                t_method = "scale"
+                t_path = self.filepaths.local_media_thumbnail(
+                    media_id, t_width, t_height, t_type, t_method
+                )
+                self._makedirs(t_path)
+                t_len = thumbnailer.scale(t_path, t_width, t_height, t_type)
+
+                local_thumbnails.append((
+                    media_id, t_width, t_height, t_type, t_method, t_len
                 ))
-            elif r_method == "crop":
-                crops.add((r_width, r_height, r_type))
 
-        for t_width, t_height, t_type in scales:
-            t_method = "scale"
-            t_path = self.filepaths.local_media_thumbnail(
-                media_id, t_width, t_height, t_type, t_method
-            )
-            self._makedirs(t_path)
-            t_len = thumbnailer.scale(t_path, t_width, t_height, t_type)
-            yield self.store.store_local_thumbnail(
-                media_id, t_width, t_height, t_type, t_method, t_len
-            )
+            for t_width, t_height, t_type in crops:
+                if (t_width, t_height, t_type) in scales:
+                    # If the aspect ratio of the cropped thumbnail matches a purely
+                    # scaled one then there is no point in calculating a separate
+                    # thumbnail.
+                    continue
+                t_method = "crop"
+                t_path = self.filepaths.local_media_thumbnail(
+                    media_id, t_width, t_height, t_type, t_method
+                )
+                self._makedirs(t_path)
+                t_len = thumbnailer.crop(t_path, t_width, t_height, t_type)
+                local_thumbnails.append((
+                    media_id, t_width, t_height, t_type, t_method, t_len
+                ))
 
-        for t_width, t_height, t_type in crops:
-            if (t_width, t_height, t_type) in scales:
-                # If the aspect ratio of the cropped thumbnail matches a purely
-                # scaled one then there is no point in calculating a separate
-                # thumbnail.
-                continue
-            t_method = "crop"
-            t_path = self.filepaths.local_media_thumbnail(
-                media_id, t_width, t_height, t_type, t_method
-            )
-            self._makedirs(t_path)
-            t_len = thumbnailer.crop(t_path, t_width, t_height, t_type)
-            yield self.store.store_local_thumbnail(
-                media_id, t_width, t_height, t_type, t_method, t_len
-            )
+        yield threads.deferToThread(generate_thumbnails)
+
+        for l in local_thumbnails:
+            yield self.store.store_local_thumbnail(*l)
 
         defer.returnValue({
             "width": m_width,

From 39e21ea51cf94f436f1f845f24c1b04f11b92c6f Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 27 Jul 2015 13:57:29 +0100
Subject: [PATCH 05/38] Add support for using keyword arguments with cached
 functions

---
 synapse/storage/_base.py     | 40 ++++++++++++++++++++++++++++++------
 synapse/storage/keys.py      |  5 ++---
 synapse/storage/push_rule.py |  8 +++-----
 synapse/storage/receipts.py  |  5 ++---
 synapse/storage/room.py      |  5 ++---
 synapse/storage/state.py     |  5 ++---
 6 files changed, 45 insertions(+), 23 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 8f812f0fd..f1265541b 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -27,6 +27,7 @@ from twisted.internet import defer
 from collections import namedtuple, OrderedDict
 
 import functools
+import inspect
 import sys
 import time
 import threading
@@ -141,13 +142,28 @@ class CacheDescriptor(object):
     which can be used to insert values into the cache specifically, without
     calling the calculation function.
     """
-    def __init__(self, orig, max_entries=1000, num_args=1, lru=False):
+    def __init__(self, orig, max_entries=1000, num_args=1, lru=False,
+                 inlineCallbacks=False):
         self.orig = orig
 
+        if inlineCallbacks:
+            self.function_to_call = defer.inlineCallbacks(orig)
+        else:
+            self.function_to_call = orig
+
         self.max_entries = max_entries
         self.num_args = num_args
         self.lru = lru
 
+        self.arg_names = inspect.getargspec(orig).args[1:num_args+1]
+
+        if len(self.arg_names) < self.num_args:
+            raise Exception(
+                "Not enough explicit positional arguments to key off of for %r."
+                " (@cached cannot key off of *args or **kwars)"
+                % (orig.__name__,)
+            )
+
     def __get__(self, obj, objtype=None):
         cache = Cache(
             name=self.orig.__name__,
@@ -158,11 +174,13 @@ class CacheDescriptor(object):
 
         @functools.wraps(self.orig)
         @defer.inlineCallbacks
-        def wrapped(*keyargs):
+        def wrapped(*args, **kwargs):
+            arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs)
+            keyargs = [arg_dict[arg_nm] for arg_nm in self.arg_names]
             try:
-                cached_result = cache.get(*keyargs[:self.num_args])
+                cached_result = cache.get(*keyargs)
                 if DEBUG_CACHES:
-                    actual_result = yield self.orig(obj, *keyargs)
+                    actual_result = yield self.function_to_call(obj, *args, **kwargs)
                     if actual_result != cached_result:
                         logger.error(
                             "Stale cache entry %s%r: cached: %r, actual %r",
@@ -177,9 +195,9 @@ class CacheDescriptor(object):
                 # while the SELECT is executing (SYN-369)
                 sequence = cache.sequence
 
-                ret = yield self.orig(obj, *keyargs)
+                ret = yield self.function_to_call(obj, *args, **kwargs)
 
-                cache.update(sequence, *keyargs[:self.num_args] + (ret,))
+                cache.update(sequence, *(keyargs + [ret]))
 
                 defer.returnValue(ret)
 
@@ -201,6 +219,16 @@ def cached(max_entries=1000, num_args=1, lru=False):
     )
 
 
+def cachedInlineCallbacks(max_entries=1000, num_args=1, lru=False):
+    return lambda orig: CacheDescriptor(
+        orig,
+        max_entries=max_entries,
+        num_args=num_args,
+        lru=lru,
+        inlineCallbacks=True,
+    )
+
+
 class LoggingTransaction(object):
     """An object that almost-transparently proxies for the 'txn' object
     passed to the constructor. Adds logging and metrics to the .execute()
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index 940a5f7e0..e3f98f0cd 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from _base import SQLBaseStore, cached
+from _base import SQLBaseStore, cachedInlineCallbacks
 
 from twisted.internet import defer
 
@@ -71,8 +71,7 @@ class KeyStore(SQLBaseStore):
             desc="store_server_certificate",
         )
 
-    @cached()
-    @defer.inlineCallbacks
+    @cachedInlineCallbacks()
     def get_all_server_verify_keys(self, server_name):
         rows = yield self._simple_select_list(
             table="server_signature_keys",
diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index 4cac118d1..a220f3632 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import SQLBaseStore, cached
+from ._base import SQLBaseStore, cachedInlineCallbacks
 from twisted.internet import defer
 
 import logging
@@ -23,8 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 class PushRuleStore(SQLBaseStore):
-    @cached()
-    @defer.inlineCallbacks
+    @cachedInlineCallbacks()
     def get_push_rules_for_user(self, user_name):
         rows = yield self._simple_select_list(
             table=PushRuleTable.table_name,
@@ -41,8 +40,7 @@ class PushRuleStore(SQLBaseStore):
 
         defer.returnValue(rows)
 
-    @cached()
-    @defer.inlineCallbacks
+    @cachedInlineCallbacks()
     def get_push_rules_enabled_for_user(self, user_name):
         results = yield self._simple_select_list(
             table=PushRuleEnableTable.table_name,
diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index 7a6af98d9..b79d6683c 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import SQLBaseStore, cached
+from ._base import SQLBaseStore, cachedInlineCallbacks
 
 from twisted.internet import defer
 
@@ -128,8 +128,7 @@ class ReceiptsStore(SQLBaseStore):
     def get_max_receipt_stream_id(self):
         return self._receipts_id_gen.get_max_token(self)
 
-    @cached
-    @defer.inlineCallbacks
+    @cachedInlineCallbacks()
     def get_graph_receipts_for_room(self, room_id):
         """Get receipts for sending to remote servers.
         """
diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index 4612a8aa8..dd5bc2c8f 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -17,7 +17,7 @@ from twisted.internet import defer
 
 from synapse.api.errors import StoreError
 
-from ._base import SQLBaseStore, cached
+from ._base import SQLBaseStore, cachedInlineCallbacks
 
 import collections
 import logging
@@ -186,8 +186,7 @@ class RoomStore(SQLBaseStore):
                 }
             )
 
-    @cached()
-    @defer.inlineCallbacks
+    @cachedInlineCallbacks()
     def get_room_name_and_aliases(self, room_id):
         def f(txn):
             sql = (
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 47bec6549..55c6d5289 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import SQLBaseStore, cached
+from ._base import SQLBaseStore, cached, cachedInlineCallbacks
 
 from twisted.internet import defer
 
@@ -189,8 +189,7 @@ class StateStore(SQLBaseStore):
         events = yield self._get_events(event_ids, get_prev_content=False)
         defer.returnValue(events)
 
-    @cached(num_args=3)
-    @defer.inlineCallbacks
+    @cachedInlineCallbacks(num_args=3)
     def get_current_state_for_key(self, room_id, event_type, state_key):
         def f(txn):
             sql = (

From a4d62ba36afc54d4e60f1371fe9b31e8b8e6834c Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Tue, 28 Jul 2015 17:34:12 +0100
Subject: [PATCH 06/38] Fix v2_alpha registration. Add unit tests.

V2 Registration forced everyone (including ASes) to create a password for a
user, when ASes should be able to omit passwords. Also unbreak AS registration
in general which checked too early if the given username was claimed by an AS;
it was checked before knowing if the AS was the one doing the registration! Add
unit tests for AS reg, user reg and disabled_registration flag.
---
 synapse/handlers/register.py                |   3 +-
 synapse/rest/client/v2_alpha/register.py    | 124 ++++++++++--------
 tests/rest/client/v2_alpha/test_register.py | 132 ++++++++++++++++++++
 3 files changed, 204 insertions(+), 55 deletions(-)
 create mode 100644 tests/rest/client/v2_alpha/test_register.py

diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index a1288b425..f81d75017 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -73,7 +73,8 @@ class RegistrationHandler(BaseHandler):
             localpart : The local part of the user ID to register. If None,
               one will be randomly generated.
             password (str) : The password to assign to this user so they can
-            login again.
+            login again. This can be None which means they cannot login again
+            via a password (e.g. the user is an application service user).
         Returns:
             A tuple of (user_id, access_token).
         Raises:
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index 0c737d73b..e1c42dd51 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -19,7 +19,7 @@ from synapse.api.constants import LoginType
 from synapse.api.errors import SynapseError, Codes
 from synapse.http.servlet import RestServlet
 
-from ._base import client_v2_pattern, parse_request_allow_empty
+from ._base import client_v2_pattern, parse_json_dict_from_request
 
 import logging
 import hmac
@@ -55,30 +55,52 @@ class RegisterRestServlet(RestServlet):
     @defer.inlineCallbacks
     def on_POST(self, request):
         yield run_on_reactor()
+        body = parse_json_dict_from_request(request)
 
-        body = parse_request_allow_empty(request)
-        # we do basic sanity checks here because the auth
-        # layer will store these in sessions
+        # we do basic sanity checks here because the auth layer will store these
+        # in sessions. Pull out the username/password provided to us.
+        desired_password = None
         if 'password' in body:
-            if ((not isinstance(body['password'], str) and
-                    not isinstance(body['password'], unicode)) or
+            if (not isinstance(body['password'], basestring) or
                     len(body['password']) > 512):
                 raise SynapseError(400, "Invalid password")
+            desired_password = body["password"]
 
+        desired_username = None
         if 'username' in body:
-            if ((not isinstance(body['username'], str) and
-                    not isinstance(body['username'], unicode)) or
+            if (not isinstance(body['username'], basestring) or
                     len(body['username']) > 512):
                 raise SynapseError(400, "Invalid username")
             desired_username = body['username']
-            yield self.registration_handler.check_username(desired_username)
 
-        is_using_shared_secret = False
-        is_application_server = False
-
-        service = None
+        appservice = None
         if 'access_token' in request.args:
-            service = yield self.auth.get_appservice_by_req(request)
+            appservice = yield self.auth.get_appservice_by_req(request)
+
+        # fork off as soon as possible for ASes and shared secret auth which
+        # have completely different registration flows to normal users
+
+        # == Application Service Registration ==
+        if appservice:
+            result = yield self._do_appservice_registration(desired_username)
+            defer.returnValue((200, result))  # we throw for non 200 responses
+            return
+
+        # == Shared Secret Registration == (e.g. create new user scripts)
+        if 'mac' in body:
+            # FIXME: Should we really be determining if this is shared secret
+            # auth based purely on the 'mac' key?
+            result = yield self._do_shared_secret_registration(
+                desired_username, desired_password, body["mac"]
+            )
+            defer.returnValue((200, result))  # we throw for non 200 responses
+            return
+
+        # == Normal User Registration == (everyone else)
+        if self.hs.config.disable_registration:
+            raise SynapseError(403, "Registration has been disabled")
+
+        yield self.registration_handler.check_username(desired_username)
 
         if self.hs.config.enable_registration_captcha:
             flows = [
@@ -91,39 +113,20 @@ class RegisterRestServlet(RestServlet):
                 [LoginType.EMAIL_IDENTITY]
             ]
 
-        result = None
-        if service:
-            is_application_server = True
-            params = body
-        elif 'mac' in body:
-            # Check registration-specific shared secret auth
-            if 'username' not in body:
-                raise SynapseError(400, "", Codes.MISSING_PARAM)
-            self._check_shared_secret_auth(
-                body['username'], body['mac']
-            )
-            is_using_shared_secret = True
-            params = body
-        else:
-            authed, result, params = yield self.auth_handler.check_auth(
-                flows, body, self.hs.get_ip_from_request(request)
-            )
-
-            if not authed:
-                defer.returnValue((401, result))
-
-        can_register = (
-            not self.hs.config.disable_registration
-            or is_application_server
-            or is_using_shared_secret
+        authed, result, params = yield self.auth_handler.check_auth(
+            flows, body, self.hs.get_ip_from_request(request)
         )
-        if not can_register:
-            raise SynapseError(403, "Registration has been disabled")
 
+        if not authed:
+            defer.returnValue((401, result))
+            return
+
+        # NB: This may be from the auth handler and NOT from the POST
         if 'password' not in params:
-            raise SynapseError(400, "", Codes.MISSING_PARAM)
-        desired_username = params['username'] if 'username' in params else None
-        new_password = params['password']
+            raise SynapseError(400, "Missing password.", Codes.MISSING_PARAM)
+
+        desired_username = params.get("username", None)
+        new_password = params.get("password", None)
 
         (user_id, token) = yield self.registration_handler.register(
             localpart=desired_username,
@@ -156,18 +159,21 @@ class RegisterRestServlet(RestServlet):
             else:
                 logger.info("bind_email not specified: not binding email")
 
-        result = {
-            "user_id": user_id,
-            "access_token": token,
-            "home_server": self.hs.hostname,
-        }
-
+        result = self._create_registration_details(user_id, token)
         defer.returnValue((200, result))
 
     def on_OPTIONS(self, _):
         return 200, {}
 
-    def _check_shared_secret_auth(self, username, mac):
+    @defer.inlineCallbacks
+    def _do_appservice_registration(self, username):
+        (user_id, token) = yield self.registration_handler.register(
+            localpart=username
+        )
+        defer.returnValue(self._create_registration_details(user_id, token))
+
+    @defer.inlineCallbacks
+    def _do_shared_secret_registration(self, username, password, mac):
         if not self.hs.config.registration_shared_secret:
             raise SynapseError(400, "Shared secret registration is not enabled")
 
@@ -183,13 +189,23 @@ class RegisterRestServlet(RestServlet):
             digestmod=sha1,
         ).hexdigest()
 
-        if compare_digest(want_mac, got_mac):
-            return True
-        else:
+        if not compare_digest(want_mac, got_mac):
             raise SynapseError(
                 403, "HMAC incorrect",
             )
 
+        (user_id, token) = yield self.registration_handler.register(
+            localpart=username, password=password
+        )
+        defer.returnValue(self._create_registration_details(user_id, token))
+
+    def _create_registration_details(self, user_id, token):
+        return {
+            "user_id": user_id,
+            "access_token": token,
+            "home_server": self.hs.hostname,
+        }
+
 
 def register_servlets(hs, http_server):
     RegisterRestServlet(hs).register(http_server)
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
new file mode 100644
index 000000000..3edc2ec2e
--- /dev/null
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -0,0 +1,132 @@
+from synapse.rest.client.v2_alpha.register import RegisterRestServlet
+from synapse.api.errors import SynapseError
+from twisted.internet import defer
+from mock import Mock, MagicMock
+from tests import unittest
+import json
+
+
+class RegisterRestServletTestCase(unittest.TestCase):
+
+    def setUp(self):
+        # do the dance to hook up request data to self.request_data
+        self.request_data = ""
+        self.request = Mock(
+            content=Mock(read=Mock(side_effect=lambda: self.request_data)),
+        )
+        self.request.args = {}
+
+        self.appservice = None
+        self.auth = Mock(get_appservice_by_req=Mock(
+            side_effect=lambda x: defer.succeed(self.appservice))
+        )
+
+        self.auth_result = (False, None, None)
+        self.auth_handler = Mock(
+            check_auth=Mock(side_effect=lambda x,y,z: self.auth_result)
+        )
+        self.registration_handler = Mock()
+        self.identity_handler = Mock()
+        self.login_handler = Mock()
+
+        # do the dance to hook it up to the hs global
+        self.handlers = Mock(
+            auth_handler=self.auth_handler,
+            registration_handler=self.registration_handler,
+            identity_handler=self.identity_handler,
+            login_handler=self.login_handler
+        )
+        self.hs = Mock()
+        self.hs.hostname = "superbig~testing~thing.com"
+        self.hs.get_auth = Mock(return_value=self.auth)
+        self.hs.get_handlers = Mock(return_value=self.handlers)
+        self.hs.config.disable_registration = False
+
+        # init the thing we're testing
+        self.servlet = RegisterRestServlet(self.hs)
+
+    @defer.inlineCallbacks
+    def test_POST_appservice_registration_valid(self):
+        user_id = "@kermit:muppet"
+        token = "kermits_access_token"
+        self.request.args = {
+            "access_token": "i_am_an_app_service"
+        }
+        self.request_data = json.dumps({
+            "username": "kermit"
+        })
+        self.appservice = {
+            "id": "1234"
+        }
+        self.registration_handler.register = Mock(return_value=(user_id, token))
+        result = yield self.servlet.on_POST(self.request)
+        self.assertEquals(result, (200, {
+            "user_id": user_id,
+            "access_token": token,
+            "home_server": self.hs.hostname
+        }))
+
+    @defer.inlineCallbacks
+    def test_POST_appservice_registration_invalid(self):
+        self.request.args = {
+            "access_token": "i_am_an_app_service"
+        }
+        self.request_data = json.dumps({
+            "username": "kermit"
+        })
+        self.appservice = None  # no application service exists
+        result = yield self.servlet.on_POST(self.request)
+        self.assertEquals(result, (401, None))
+
+    def test_POST_bad_password(self):
+        self.request_data = json.dumps({
+            "username": "kermit",
+            "password": 666
+        })
+        d = self.servlet.on_POST(self.request)
+        return self.assertFailure(d, SynapseError)
+
+    def test_POST_bad_username(self):
+        self.request_data = json.dumps({
+            "username": 777,
+            "password": "monkey"
+        })
+        d = self.servlet.on_POST(self.request)
+        return self.assertFailure(d, SynapseError)
+
+    @defer.inlineCallbacks
+    def test_POST_user_valid(self):
+        user_id = "@kermit:muppet"
+        token = "kermits_access_token"
+        self.request_data = json.dumps({
+            "username": "kermit",
+            "password": "monkey"
+        })
+        self.registration_handler.check_username = Mock(return_value=True)
+        self.auth_result = (True, None, {
+            "username": "kermit",
+            "password": "monkey"
+        })
+        self.registration_handler.register = Mock(return_value=(user_id, token))
+
+        result = yield self.servlet.on_POST(self.request)
+        self.assertEquals(result, (200, {
+            "user_id": user_id,
+            "access_token": token,
+            "home_server": self.hs.hostname
+        }))
+
+    def test_POST_disabled_registration(self):
+        self.hs.config.disable_registration = True
+        self.request_data = json.dumps({
+            "username": "kermit",
+            "password": "monkey"
+        })
+        self.registration_handler.check_username = Mock(return_value=True)
+        self.auth_result = (True, None, {
+            "username": "kermit",
+            "password": "monkey"
+        })
+        self.registration_handler.register = Mock(return_value=("@user:id", "t"))
+        d = self.servlet.on_POST(self.request)
+        return self.assertFailure(d, SynapseError)
\ No newline at end of file

From c772dffc9f024ce3135039c9f8032f09cc730841 Mon Sep 17 00:00:00 2001
From: Matthew Hodgson <matthew@matrix.org>
Date: Wed, 29 Jul 2015 09:39:55 +0100
Subject: [PATCH 07/38] improve OS X instructions and remove all the leading
 $'s to make it easier to c+p commands

---
 README.rst | 92 ++++++++++++++++++++++++++++--------------------------
 1 file changed, 47 insertions(+), 45 deletions(-)

diff --git a/README.rst b/README.rst
index 21e347968..5ff53f2df 100644
--- a/README.rst
+++ b/README.rst
@@ -101,25 +101,26 @@ header files for python C extensions.
 
 Installing prerequisites on Ubuntu or Debian::
 
-    $ sudo apt-get install build-essential python2.7-dev libffi-dev \
-                           python-pip python-setuptools sqlite3 \
-                           libssl-dev python-virtualenv libjpeg-dev
+    sudo apt-get install build-essential python2.7-dev libffi-dev \
+                         python-pip python-setuptools sqlite3 \
+                         libssl-dev python-virtualenv libjpeg-dev
                            
 Installing prerequisites on ArchLinux::
 
-    $ sudo pacman -S base-devel python2 python-pip \
-                     python-setuptools python-virtualenv sqlite3
+    sudo pacman -S base-devel python2 python-pip \
+                   python-setuptools python-virtualenv sqlite3
 
 Installing prerequisites on Mac OS X::
 
-    $ xcode-select --install
-    $ sudo pip install virtualenv
+    xcode-select --install
+    sudo easy_install pip
+    sudo pip install virtualenv
     
 To install the synapse homeserver run::
 
-    $ virtualenv -p python2.7 ~/.synapse
-    $ source ~/.synapse/bin/activate
-    $ pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master
+    virtualenv -p python2.7 ~/.synapse
+    source ~/.synapse/bin/activate
+    pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master
 
 This installs synapse, along with the libraries it uses, into a virtual
 environment under ``~/.synapse``.  Feel free to pick a different directory
@@ -132,8 +133,8 @@ above in Docker at https://registry.hub.docker.com/u/silviof/docker-matrix/.
 
 To set up your homeserver, run (in your virtualenv, as before)::
 
-    $ cd ~/.synapse
-    $ python -m synapse.app.homeserver \
+    cd ~/.synapse
+    python -m synapse.app.homeserver \
         --server-name machine.my.domain.name \
         --config-path homeserver.yaml \
         --generate-config
@@ -192,9 +193,9 @@ Running Synapse
 To actually run your new homeserver, pick a working directory for Synapse to run 
 (e.g. ``~/.synapse``), and::
 
-    $ cd ~/.synapse
-    $ source ./bin/activate
-    $ synctl start
+    cd ~/.synapse
+    source ./bin/activate
+    synctl start
 
 Platform Specific Instructions
 ==============================
@@ -212,12 +213,12 @@ defaults to python 3, but synapse currently assumes python 2.7 by default:
 
 pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 )::
 
-    $ sudo pip2.7 install --upgrade pip
+    sudo pip2.7 install --upgrade pip
     
 You also may need to explicitly specify python 2.7 again during the install
 request::
 
-    $ pip2.7 install --process-dependency-links \
+    pip2.7 install --process-dependency-links \
         https://github.com/matrix-org/synapse/tarball/master
     
 If you encounter an error with lib bcrypt causing an Wrong ELF Class:
@@ -225,13 +226,13 @@ ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly
 compile it under the right architecture. (This should not be needed if
 installing under virtualenv)::
 
-    $ sudo pip2.7 uninstall py-bcrypt
-    $ sudo pip2.7 install py-bcrypt
+    sudo pip2.7 uninstall py-bcrypt
+    sudo pip2.7 install py-bcrypt
     
 During setup of Synapse you need to call python2.7 directly again::
 
-    $ cd ~/.synapse
-    $ python2.7 -m synapse.app.homeserver \
+    cd ~/.synapse
+    python2.7 -m synapse.app.homeserver \
       --server-name machine.my.domain.name \
       --config-path homeserver.yaml \
       --generate-config
@@ -279,22 +280,22 @@ Synapse requires pip 1.7 or later, so if your OS provides too old a version and
 you get errors about ``error: no such option: --process-dependency-links`` you 
 may need to manually upgrade it::
 
-    $ sudo pip install --upgrade pip
+    sudo pip install --upgrade pip
 
 If pip crashes mid-installation for reason (e.g. lost terminal), pip may
 refuse to run until you remove the temporary installation directory it
 created. To reset the installation::
 
-    $ rm -rf /tmp/pip_install_matrix
+    rm -rf /tmp/pip_install_matrix
 
 pip seems to leak *lots* of memory during installation.  For instance, a Linux 
 host with 512MB of RAM may run out of memory whilst installing Twisted.  If this 
 happens, you will have to individually install the dependencies which are 
 failing, e.g.::
 
-    $ pip install twisted
+    pip install twisted
 
-On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you
+On OS X, if you encounter clang: error: unknown argument: '-mno-fused-madd' you
 will need to export CFLAGS=-Qunused-arguments.
 
 Troubleshooting Running
@@ -310,10 +311,11 @@ correctly, causing all tests to fail with errors about missing "sodium.h". To
 fix try re-installing from PyPI or directly from
 (https://github.com/pyca/pynacl)::
 
-    $ # Install from PyPI
-    $ pip install --user --upgrade --force pynacl
-    $ # Install from github
-    $ pip install --user https://github.com/pyca/pynacl/tarball/master
+    # Install from PyPI
+    pip install --user --upgrade --force pynacl
+    
+    # Install from github
+    pip install --user https://github.com/pyca/pynacl/tarball/master
 
 ArchLinux
 ~~~~~~~~~
@@ -321,7 +323,7 @@ ArchLinux
 If running `$ synctl start` fails with 'returned non-zero exit status 1',
 you will need to explicitly call Python2.7 - either running as::
 
-    $ python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml
+    python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml
     
 ...or by editing synctl with the correct python executable.
 
@@ -331,16 +333,16 @@ Synapse Development
 To check out a synapse for development, clone the git repo into a working
 directory of your choice::
 
-    $ git clone https://github.com/matrix-org/synapse.git
-    $ cd synapse
+    git clone https://github.com/matrix-org/synapse.git
+    cd synapse
 
 Synapse has a number of external dependencies, that are easiest
 to install using pip and a virtualenv::
 
-    $ virtualenv env
-    $ source env/bin/activate
-    $ python synapse/python_dependencies.py | xargs -n1 pip install
-    $ pip install setuptools_trial mock
+    virtualenv env
+    source env/bin/activate
+    python synapse/python_dependencies.py | xargs -n1 pip install
+    pip install setuptools_trial mock
 
 This will run a process of downloading and installing all the needed
 dependencies into a virtual env.
@@ -348,7 +350,7 @@ dependencies into a virtual env.
 Once this is done, you may wish to run Synapse's unit tests, to
 check that everything is installed as it should be::
 
-    $ python setup.py test
+    python setup.py test
 
 This should end with a 'PASSED' result::
 
@@ -389,11 +391,11 @@ IDs:
 For the first form, simply pass the required hostname (of the machine) as the
 --server-name parameter::
 
-    $ python -m synapse.app.homeserver \
+    python -m synapse.app.homeserver \
         --server-name machine.my.domain.name \
         --config-path homeserver.yaml \
         --generate-config
-    $ python -m synapse.app.homeserver --config-path homeserver.yaml
+    python -m synapse.app.homeserver --config-path homeserver.yaml
 
 Alternatively, you can run ``synctl start`` to guide you through the process.
 
@@ -410,11 +412,11 @@ record would then look something like::
 At this point, you should then run the homeserver with the hostname of this
 SRV record, as that is the name other machines will expect it to have::
 
-    $ python -m synapse.app.homeserver \
+    python -m synapse.app.homeserver \
         --server-name YOURDOMAIN \
         --config-path homeserver.yaml \
         --generate-config
-    $ python -m synapse.app.homeserver --config-path homeserver.yaml
+    python -m synapse.app.homeserver --config-path homeserver.yaml
 
 
 You may additionally want to pass one or more "-v" options, in order to
@@ -428,7 +430,7 @@ private federation (``localhost:8080``, ``localhost:8081`` and
 ``localhost:8082``) which you can then access through the webclient running at
 http://localhost:8080. Simply run::
 
-    $ demo/start.sh
+    demo/start.sh
     
 This is mainly useful just for development purposes.
 
@@ -502,10 +504,10 @@ Building Internal API Documentation
 Before building internal API documentation install sphinx and
 sphinxcontrib-napoleon::
 
-    $ pip install sphinx
-    $ pip install sphinxcontrib-napoleon
+    pip install sphinx
+    pip install sphinxcontrib-napoleon
 
 Building internal API documentation::
 
-    $ python setup.py build_sphinx
+    python setup.py build_sphinx
 

From 11b0a3407485e98082bf06d771e5ae2f68106ca7 Mon Sep 17 00:00:00 2001
From: Kegan Dougal <kegan@matrix.org>
Date: Wed, 29 Jul 2015 10:00:54 +0100
Subject: [PATCH 08/38] Use the same reg paths as register v1 for ASes.

Namely this means using registration_handler.appservice_register.
---
 synapse/rest/client/v2_alpha/register.py    | 10 ++++++----
 tests/rest/client/v2_alpha/test_register.py |  4 +++-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index e1c42dd51..cf54e1dac 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -82,7 +82,9 @@ class RegisterRestServlet(RestServlet):
 
         # == Application Service Registration ==
         if appservice:
-            result = yield self._do_appservice_registration(desired_username)
+            result = yield self._do_appservice_registration(
+                desired_username, request.args["access_token"][0]
+            )
             defer.returnValue((200, result))  # we throw for non 200 responses
             return
 
@@ -166,9 +168,9 @@ class RegisterRestServlet(RestServlet):
         return 200, {}
 
     @defer.inlineCallbacks
-    def _do_appservice_registration(self, username):
-        (user_id, token) = yield self.registration_handler.register(
-            localpart=username
+    def _do_appservice_registration(self, username, as_token):
+        (user_id, token) = yield self.registration_handler.appservice_register(
+            username, as_token
         )
         defer.returnValue(self._create_registration_details(user_id, token))
 
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
index 3edc2ec2e..66fd25964 100644
--- a/tests/rest/client/v2_alpha/test_register.py
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -58,7 +58,9 @@ class RegisterRestServletTestCase(unittest.TestCase):
         self.appservice = {
             "id": "1234"
         }
-        self.registration_handler.register = Mock(return_value=(user_id, token))
+        self.registration_handler.appservice_register = Mock(
+            return_value=(user_id, token)
+        )
         result = yield self.servlet.on_POST(self.request)
         self.assertEquals(result, (200, {
             "user_id": user_id,

From 28d07a02e42c90732b708de870af2f9b21ba13f0 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Mon, 3 Aug 2015 15:31:21 +0100
Subject: [PATCH 09/38] Add vector.im as trusted ID server

---
 synapse/handlers/identity.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 6200e1077..c1095708a 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -44,7 +44,7 @@ class IdentityHandler(BaseHandler):
         http_client = SimpleHttpClient(self.hs)
         # XXX: make this configurable!
         # trustedIdServers = ['matrix.org', 'localhost:8090']
-        trustedIdServers = ['matrix.org']
+        trustedIdServers = ['matrix.org', 'vector.im']
 
         if 'id_server' in creds:
             id_server = creds['id_server']

From 7148aaf5d0f75c463c93ac69885d05160fee4d4a Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Mon, 3 Aug 2015 17:03:27 +0100
Subject: [PATCH 10/38] Don't try & check the username if we don't have one
 (which we won't if it's been saved in the auth layer)

---
 synapse/rest/client/v2_alpha/register.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index cf54e1dac..b5926f9ca 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -102,7 +102,8 @@ class RegisterRestServlet(RestServlet):
         if self.hs.config.disable_registration:
             raise SynapseError(403, "Registration has been disabled")
 
-        yield self.registration_handler.check_username(desired_username)
+        if desired_username is not None:
+            yield self.registration_handler.check_username(desired_username)
 
         if self.hs.config.enable_registration_captcha:
             flows = [

From a89559d79714b8dd07ec5a73f2629d003fcad92c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 5 Aug 2015 15:39:47 +0100
Subject: [PATCH 11/38] Use LRU cache by default

---
 synapse/storage/_base.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 8f812f0fd..2a601e37e 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -57,7 +57,7 @@ cache_counter = metrics.register_cache(
 
 class Cache(object):
 
-    def __init__(self, name, max_entries=1000, keylen=1, lru=False):
+    def __init__(self, name, max_entries=1000, keylen=1, lru=True):
         if lru:
             self.cache = LruCache(max_size=max_entries)
             self.max_entries = None
@@ -141,7 +141,7 @@ class CacheDescriptor(object):
     which can be used to insert values into the cache specifically, without
     calling the calculation function.
     """
-    def __init__(self, orig, max_entries=1000, num_args=1, lru=False):
+    def __init__(self, orig, max_entries=1000, num_args=1, lru=True):
         self.orig = orig
 
         self.max_entries = max_entries
@@ -192,7 +192,7 @@ class CacheDescriptor(object):
         return wrapped
 
 
-def cached(max_entries=1000, num_args=1, lru=False):
+def cached(max_entries=1000, num_args=1, lru=True):
     return lambda orig: CacheDescriptor(
         orig,
         max_entries=max_entries,

From 1e62a3d3a9d09fbd856b4abd733f4fb687f746e0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 5 Aug 2015 15:40:40 +0100
Subject: [PATCH 12/38] Up the cache size for 'get_joined_hosts_for_room' and
 'get_users_in_room'

---
 synapse/storage/roommember.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 4db07f6fb..55dd3f6cf 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -78,7 +78,7 @@ class RoomMemberStore(SQLBaseStore):
             lambda events: events[0] if events else None
         )
 
-    @cached()
+    @cached(max_entries=5000)
     def get_users_in_room(self, room_id):
         def f(txn):
 
@@ -154,7 +154,7 @@ class RoomMemberStore(SQLBaseStore):
             RoomsForUser(**r) for r in self.cursor_to_dict(txn)
         ]
 
-    @cached()
+    @cached(max_entries=5000)
     def get_joined_hosts_for_room(self, room_id):
         return self.runInteraction(
             "get_joined_hosts_for_room",

From 7eea3e356ff58168f3525879a8eb684f0681ee68 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 Aug 2015 13:33:34 +0100
Subject: [PATCH 13/38] Make @cached cache deferreds rather than the deferreds'
 values

---
 synapse/storage/_base.py    | 21 ++++++++-------------
 synapse/util/async.py       |  9 +++++++--
 tests/storage/test__base.py | 11 +++++++----
 3 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index f1265541b..8604d38c3 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -15,6 +15,7 @@
 import logging
 
 from synapse.api.errors import StoreError
+from synapse.util.async import ObservableDeferred
 from synapse.util.logutils import log_function
 from synapse.util.logcontext import preserve_context_over_fn, LoggingContext
 from synapse.util.lrucache import LruCache
@@ -173,33 +174,27 @@ class CacheDescriptor(object):
         )
 
         @functools.wraps(self.orig)
-        @defer.inlineCallbacks
         def wrapped(*args, **kwargs):
             arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs)
             keyargs = [arg_dict[arg_nm] for arg_nm in self.arg_names]
             try:
                 cached_result = cache.get(*keyargs)
-                if DEBUG_CACHES:
-                    actual_result = yield self.function_to_call(obj, *args, **kwargs)
-                    if actual_result != cached_result:
-                        logger.error(
-                            "Stale cache entry %s%r: cached: %r, actual %r",
-                            self.orig.__name__, keyargs,
-                            cached_result, actual_result,
-                        )
-                        raise ValueError("Stale cache entry")
-                defer.returnValue(cached_result)
+                return cached_result.observe()
             except KeyError:
                 # Get the sequence number of the cache before reading from the
                 # database so that we can tell if the cache is invalidated
                 # while the SELECT is executing (SYN-369)
                 sequence = cache.sequence
 
-                ret = yield self.function_to_call(obj, *args, **kwargs)
+                ret = defer.maybeDeferred(
+                    self.function_to_call,
+                    obj, *args, **kwargs
+                )
+                ret = ObservableDeferred(ret, consumeErrors=False)
 
                 cache.update(sequence, *(keyargs + [ret]))
 
-                defer.returnValue(ret)
+                return ret.observe()
 
         wrapped.invalidate = cache.invalidate
         wrapped.invalidate_all = cache.invalidate_all
diff --git a/synapse/util/async.py b/synapse/util/async.py
index 5a1d545c9..7bf2d38bb 100644
--- a/synapse/util/async.py
+++ b/synapse/util/async.py
@@ -51,7 +51,7 @@ class ObservableDeferred(object):
         object.__setattr__(self, "_observers", set())
 
         def callback(r):
-            self._result = (True, r)
+            object.__setattr__(self, "_result", (True, r))
             while self._observers:
                 try:
                     self._observers.pop().callback(r)
@@ -60,7 +60,7 @@ class ObservableDeferred(object):
             return r
 
         def errback(f):
-            self._result = (False, f)
+            object.__setattr__(self, "_result", (False, f))
             while self._observers:
                 try:
                     self._observers.pop().errback(f)
@@ -97,3 +97,8 @@ class ObservableDeferred(object):
 
     def __setattr__(self, name, value):
         setattr(self._deferred, name, value)
+
+    def __repr__(self):
+        return "<ObservableDeferred object at %s, result=%r, _deferred=%r>" % (
+            id(self), self._result, self._deferred,
+        )
diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py
index 8c3d2952b..8fa305d18 100644
--- a/tests/storage/test__base.py
+++ b/tests/storage/test__base.py
@@ -17,6 +17,8 @@
 from tests import unittest
 from twisted.internet import defer
 
+from synapse.util.async import ObservableDeferred
+
 from synapse.storage._base import Cache, cached
 
 
@@ -178,19 +180,20 @@ class CacheDecoratorTestCase(unittest.TestCase):
         self.assertTrue(callcount[0] >= 14,
             msg="Expected callcount >= 14, got %d" % (callcount[0]))
 
-    @defer.inlineCallbacks
     def test_prefill(self):
         callcount = [0]
 
+        d = defer.succeed(123)
+
         class A(object):
             @cached()
             def func(self, key):
                 callcount[0] += 1
-                return key
+                return d
 
         a = A()
 
-        a.func.prefill("foo", 123)
+        a.func.prefill("foo", ObservableDeferred(d))
 
-        self.assertEquals((yield a.func("foo")), 123)
+        self.assertEquals(a.func("foo").result, d.result)
         self.assertEquals(callcount[0], 0)

From 433314cc34295ffefca349b9fc6914d81f2521e0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 Aug 2015 14:01:05 +0100
Subject: [PATCH 14/38] Re-implement DEBUG_CACHES flag

---
 synapse/storage/_base.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 7b2be6745..8384299a1 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -178,8 +178,23 @@ class CacheDescriptor(object):
             arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs)
             keyargs = [arg_dict[arg_nm] for arg_nm in self.arg_names]
             try:
-                cached_result = cache.get(*keyargs)
-                return cached_result.observe()
+                cached_result_d = cache.get(*keyargs)
+
+                if DEBUG_CACHES:
+
+                    @defer.inlineCallbacks
+                    def check_result(cached_result):
+                        actual_result = yield self.function_to_call(obj, *args, **kwargs)
+                        if actual_result != cached_result:
+                            logger.error(
+                                "Stale cache entry %s%r: cached: %r, actual %r",
+                                self.orig.__name__, keyargs,
+                                cached_result, actual_result,
+                            )
+                            raise ValueError("Stale cache entry")
+                    cached_result_d.observe().addCallback(check_result)
+
+                return cached_result_d.observe()
             except KeyError:
                 # Get the sequence number of the cache before reading from the
                 # database so that we can tell if the cache is invalidated

From b811c9857491c0569ae367708721fbbaebf3adab Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 Aug 2015 14:01:27 +0100
Subject: [PATCH 15/38] Remove failed deferreds from cache

---
 synapse/storage/_base.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 8384299a1..99c094875 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -205,8 +205,14 @@ class CacheDescriptor(object):
                     self.function_to_call,
                     obj, *args, **kwargs
                 )
-                ret = ObservableDeferred(ret, consumeErrors=False)
 
+                def onErr(f):
+                    cache.invalidate(*keyargs)
+                    return f
+
+                ret.addErrback(onErr)
+
+                ret = ObservableDeferred(ret, consumeErrors=False)
                 cache.update(sequence, *(keyargs + [ret]))
 
                 return ret.observe()

From 63b1eaf32c57f0e1bae9b6a3768a560165637aee Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 Aug 2015 14:02:50 +0100
Subject: [PATCH 16/38] Docs

---
 synapse/storage/_base.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 99c094875..0872a438f 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -132,6 +132,9 @@ class Cache(object):
 class CacheDescriptor(object):
     """ A method decorator that applies a memoizing cache around the function.
 
+    This caches deferreds, rather than the results themselves. Deferreds that
+    fail are removed from the cache.
+
     The function is presumed to take zero or more arguments, which are used in
     a tuple as the key for the cache. Hits are served directly from the cache;
     misses use the function body to generate the value.

From efe60d5e8c76cede325aa09a138f5033b90b8d90 Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Fri, 7 Aug 2015 16:36:42 +0100
Subject: [PATCH 17/38] Only print the pidfile path on startup if requested by
 a commandline flag

---
 synapse/app/homeserver.py | 3 ++-
 synapse/config/server.py  | 7 +++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 49e27c9e1..f04493f92 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -657,7 +657,8 @@ def run(hs):
 
     if hs.config.daemonize:
 
-        print hs.config.pid_file
+        if hs.config.print_pidfile:
+            print hs.config.pid_file
 
         daemon = Daemonize(
             app="synapse-homeserver",
diff --git a/synapse/config/server.py b/synapse/config/server.py
index f4d4a8710..f9a3b5f15 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -24,6 +24,7 @@ class ServerConfig(Config):
         self.web_client = config["web_client"]
         self.soft_file_limit = config["soft_file_limit"]
         self.daemonize = config.get("daemonize")
+        self.print_pidfile = config.get("print_pidfile")
         self.use_frozen_dicts = config.get("use_frozen_dicts", True)
 
         self.listeners = config.get("listeners", [])
@@ -208,12 +209,18 @@ class ServerConfig(Config):
             self.manhole = args.manhole
         if args.daemonize is not None:
             self.daemonize = args.daemonize
+        if args.print_pidfile is not None:
+            self.print_pidfile = args.print_pidfile
 
     def add_arguments(self, parser):
         server_group = parser.add_argument_group("server")
         server_group.add_argument("-D", "--daemonize", action='store_true',
                                   default=None,
                                   help="Daemonize the home server")
+        server_group.add_argument("--print-pidfile", action='store_true',
+                                  default=None,
+                                  help="Print the path to the pidfile just"
+                                  " before daemonizing")
         server_group.add_argument("--manhole", metavar="PORT", dest="manhole",
                                   type=int,
                                   help="Turn on the twisted telnet manhole"

From e3c8e2c13c7c99a10d52a984c0bffa5715868c9a Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Fri, 7 Aug 2015 16:42:27 +0100
Subject: [PATCH 18/38] Add a --generate-keys option

---
 synapse/config/_base.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index d483c67c6..c408db2b4 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -138,6 +138,11 @@ class Config(object):
             action="store_true",
             help="Generate a config file for the server name"
         )
+        config_parser.add_argument(
+            "--generate-keys",
+            action="store_true",
+            help="Generate any missing key files then exit"
+        )
         config_parser.add_argument(
             "-H", "--server-name",
             help="The server name to generate a config file for"
@@ -230,4 +235,8 @@ class Config(object):
 
         obj.invoke_all("read_arguments", args)
 
+        if config_args.generate_keys:
+            obj.invoke_all("generate_files", config)
+            sys.exit(0)
+
         return obj

From 0db40d3e939506e22b927214bb0d3eaad40730ab Mon Sep 17 00:00:00 2001
From: "Paul \"LeoNerd\" Evans" <paul@matrix.org>
Date: Fri, 7 Aug 2015 17:22:11 +0100
Subject: [PATCH 19/38] Don't complain about extra .pyc files we find while
 hunting for database schemas

---
 synapse/storage/__init__.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 71d5d9250..973e78e04 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -354,6 +354,11 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files,
                     )
                 logger.debug("Running script %s", relative_path)
                 module.run_upgrade(cur, database_engine)
+            elif ext == ".pyc":
+                # Sometimes .pyc files turn up anyway even though we've
+                # disabled their generation; e.g. from distribution package
+                # installers. Silently skip it
+                pass
             elif ext == ".sql":
                 # A plain old .sql file, just read and execute it
                 logger.debug("Applying schema %s", relative_path)

From 20addfa358e4f72b9f0c25d48c1e3ecfc08a68b2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 7 Aug 2015 11:52:21 +0100
Subject: [PATCH 20/38] Change Cache to not use *args in its interface

---
 synapse/storage/__init__.py         |  4 +-
 synapse/storage/_base.py            | 61 +++++++++++++++--------------
 synapse/storage/directory.py        |  4 +-
 synapse/storage/event_federation.py |  4 +-
 synapse/storage/events.py           | 21 +++++-----
 synapse/storage/keys.py             |  2 +-
 synapse/storage/presence.py         |  4 +-
 synapse/storage/push_rule.py        | 16 ++++----
 synapse/storage/registration.py     |  2 +-
 synapse/storage/roommember.py       |  6 +--
 tests/storage/test__base.py         | 12 +++---
 11 files changed, 69 insertions(+), 67 deletions(-)

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 71d5d9250..1a6a8a376 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -99,7 +99,7 @@ class DataStore(RoomMemberStore, RoomStore,
         key = (user.to_string(), access_token, device_id, ip)
 
         try:
-            last_seen = self.client_ip_last_seen.get(*key)
+            last_seen = self.client_ip_last_seen.get(key)
         except KeyError:
             last_seen = None
 
@@ -107,7 +107,7 @@ class DataStore(RoomMemberStore, RoomStore,
         if last_seen is not None and (now - last_seen) < LAST_SEEN_GRANULARITY:
             defer.returnValue(None)
 
-        self.client_ip_last_seen.prefill(*key + (now,))
+        self.client_ip_last_seen.prefill(key, now)
 
         # It's safe not to lock here: a) no unique constraint,
         # b) LAST_SEEN_GRANULARITY makes concurrent updates incredibly unlikely
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 0872a438f..e07cf3b58 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -57,6 +57,9 @@ cache_counter = metrics.register_cache(
 )
 
 
+_CacheSentinel = object()
+
+
 class Cache(object):
 
     def __init__(self, name, max_entries=1000, keylen=1, lru=True):
@@ -83,41 +86,38 @@ class Cache(object):
                     "Cache objects can only be accessed from the main thread"
                 )
 
-    def get(self, *keyargs):
-        if len(keyargs) != self.keylen:
-            raise ValueError("Expected a key to have %d items", self.keylen)
-
-        if keyargs in self.cache:
+    def get(self, keyargs, default=_CacheSentinel):
+        val = self.cache.get(keyargs, _CacheSentinel)
+        if val is not _CacheSentinel:
             cache_counter.inc_hits(self.name)
-            return self.cache[keyargs]
+            return val
 
         cache_counter.inc_misses(self.name)
-        raise KeyError()
 
-    def update(self, sequence, *args):
+        if default is _CacheSentinel:
+            raise KeyError()
+        else:
+            return default
+
+    def update(self, sequence, keyargs, value):
         self.check_thread()
         if self.sequence == sequence:
             # Only update the cache if the caches sequence number matches the
             # number that the cache had before the SELECT was started (SYN-369)
-            self.prefill(*args)
-
-    def prefill(self, *args):  # because I can't  *keyargs, value
-        keyargs = args[:-1]
-        value = args[-1]
-
-        if len(keyargs) != self.keylen:
-            raise ValueError("Expected a key to have %d items", self.keylen)
+            self.prefill(keyargs, value)
 
+    def prefill(self, keyargs, value):
         if self.max_entries is not None:
             while len(self.cache) >= self.max_entries:
                 self.cache.popitem(last=False)
 
         self.cache[keyargs] = value
 
-    def invalidate(self, *keyargs):
+    def invalidate(self, keyargs):
         self.check_thread()
-        if len(keyargs) != self.keylen:
-            raise ValueError("Expected a key to have %d items", self.keylen)
+        if not isinstance(keyargs, tuple):
+            raise ValueError("keyargs must be a tuple.")
+
         # Increment the sequence number so that any SELECT statements that
         # raced with the INSERT don't update the cache (SYN-369)
         self.sequence += 1
@@ -168,20 +168,21 @@ class CacheDescriptor(object):
                 % (orig.__name__,)
             )
 
-    def __get__(self, obj, objtype=None):
-        cache = Cache(
+        self.cache = Cache(
             name=self.orig.__name__,
             max_entries=self.max_entries,
             keylen=self.num_args,
             lru=self.lru,
         )
 
+    def __get__(self, obj, objtype=None):
+
         @functools.wraps(self.orig)
         def wrapped(*args, **kwargs):
             arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs)
-            keyargs = [arg_dict[arg_nm] for arg_nm in self.arg_names]
+            keyargs = tuple(arg_dict[arg_nm] for arg_nm in self.arg_names)
             try:
-                cached_result_d = cache.get(*keyargs)
+                cached_result_d = self.cache.get(keyargs)
 
                 if DEBUG_CACHES:
 
@@ -202,7 +203,7 @@ class CacheDescriptor(object):
                 # Get the sequence number of the cache before reading from the
                 # database so that we can tell if the cache is invalidated
                 # while the SELECT is executing (SYN-369)
-                sequence = cache.sequence
+                sequence = self.cache.sequence
 
                 ret = defer.maybeDeferred(
                     self.function_to_call,
@@ -210,19 +211,19 @@ class CacheDescriptor(object):
                 )
 
                 def onErr(f):
-                    cache.invalidate(*keyargs)
+                    self.cache.invalidate(keyargs)
                     return f
 
                 ret.addErrback(onErr)
 
-                ret = ObservableDeferred(ret, consumeErrors=False)
-                cache.update(sequence, *(keyargs + [ret]))
+                ret = ObservableDeferred(ret, consumeErrors=True)
+                self.cache.update(sequence, keyargs, ret)
 
                 return ret.observe()
 
-        wrapped.invalidate = cache.invalidate
-        wrapped.invalidate_all = cache.invalidate_all
-        wrapped.prefill = cache.prefill
+        wrapped.invalidate = self.cache.invalidate
+        wrapped.invalidate_all = self.cache.invalidate_all
+        wrapped.prefill = self.cache.prefill
 
         obj.__dict__[self.orig.__name__] = wrapped
 
diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py
index 2b2bdf861..f3947bbe8 100644
--- a/synapse/storage/directory.py
+++ b/synapse/storage/directory.py
@@ -104,7 +104,7 @@ class DirectoryStore(SQLBaseStore):
                 },
                 desc="create_room_alias_association",
             )
-        self.get_aliases_for_room.invalidate(room_id)
+        self.get_aliases_for_room.invalidate((room_id,))
 
     @defer.inlineCallbacks
     def delete_room_alias(self, room_alias):
@@ -114,7 +114,7 @@ class DirectoryStore(SQLBaseStore):
             room_alias,
         )
 
-        self.get_aliases_for_room.invalidate(room_id)
+        self.get_aliases_for_room.invalidate((room_id,))
         defer.returnValue(room_id)
 
     def _delete_room_alias_txn(self, txn, room_alias):
diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 45b86c94e..910b6598a 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -362,7 +362,7 @@ class EventFederationStore(SQLBaseStore):
 
         for room_id in events_by_room:
             txn.call_after(
-                self.get_latest_event_ids_in_room.invalidate, room_id
+                self.get_latest_event_ids_in_room.invalidate, (room_id,)
             )
 
     def get_backfill_events(self, room_id, event_list, limit):
@@ -505,4 +505,4 @@ class EventFederationStore(SQLBaseStore):
         query = "DELETE FROM event_forward_extremities WHERE room_id = ?"
 
         txn.execute(query, (room_id,))
-        txn.call_after(self.get_latest_event_ids_in_room.invalidate, room_id)
+        txn.call_after(self.get_latest_event_ids_in_room.invalidate, (room_id,))
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index ed7ea3880..5b6491802 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -162,8 +162,8 @@ class EventsStore(SQLBaseStore):
         if current_state:
             txn.call_after(self.get_current_state_for_key.invalidate_all)
             txn.call_after(self.get_rooms_for_user.invalidate_all)
-            txn.call_after(self.get_users_in_room.invalidate, event.room_id)
-            txn.call_after(self.get_joined_hosts_for_room.invalidate, event.room_id)
+            txn.call_after(self.get_users_in_room.invalidate, (event.room_id,))
+            txn.call_after(self.get_joined_hosts_for_room.invalidate, (event.room_id,))
             txn.call_after(self.get_room_name_and_aliases, event.room_id)
 
             self._simple_delete_txn(
@@ -430,13 +430,13 @@ class EventsStore(SQLBaseStore):
                 if not context.rejected:
                     txn.call_after(
                         self.get_current_state_for_key.invalidate,
-                        event.room_id, event.type, event.state_key
-                        )
+                        (event.room_id, event.type, event.state_key,)
+                    )
 
                     if event.type in [EventTypes.Name, EventTypes.Aliases]:
                         txn.call_after(
                             self.get_room_name_and_aliases.invalidate,
-                            event.room_id
+                            (event.room_id,)
                         )
 
                     self._simple_upsert_txn(
@@ -567,8 +567,9 @@ class EventsStore(SQLBaseStore):
     def _invalidate_get_event_cache(self, event_id):
         for check_redacted in (False, True):
             for get_prev_content in (False, True):
-                self._get_event_cache.invalidate(event_id, check_redacted,
-                                                 get_prev_content)
+                self._get_event_cache.invalidate(
+                    (event_id, check_redacted, get_prev_content)
+                )
 
     def _get_event_txn(self, txn, event_id, check_redacted=True,
                        get_prev_content=False, allow_rejected=False):
@@ -589,7 +590,7 @@ class EventsStore(SQLBaseStore):
         for event_id in events:
             try:
                 ret = self._get_event_cache.get(
-                    event_id, check_redacted, get_prev_content
+                    (event_id, check_redacted, get_prev_content,)
                 )
 
                 if allow_rejected or not ret.rejected_reason:
@@ -822,7 +823,7 @@ class EventsStore(SQLBaseStore):
                 ev.unsigned["prev_content"] = prev.get_dict()["content"]
 
         self._get_event_cache.prefill(
-            ev.event_id, check_redacted, get_prev_content, ev
+            (ev.event_id, check_redacted, get_prev_content), ev
         )
 
         defer.returnValue(ev)
@@ -879,7 +880,7 @@ class EventsStore(SQLBaseStore):
                 ev.unsigned["prev_content"] = prev.get_dict()["content"]
 
         self._get_event_cache.prefill(
-            ev.event_id, check_redacted, get_prev_content, ev
+            (ev.event_id, check_redacted, get_prev_content), ev
         )
 
         return ev
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index e3f98f0cd..49b8e37cf 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -131,7 +131,7 @@ class KeyStore(SQLBaseStore):
             desc="store_server_verify_key",
         )
 
-        self.get_all_server_verify_keys.invalidate(server_name)
+        self.get_all_server_verify_keys.invalidate((server_name,))
 
     def store_server_keys_json(self, server_name, key_id, from_server,
                                ts_now_ms, ts_expires_ms, key_json_bytes):
diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py
index fefcf6bce..576cf670c 100644
--- a/synapse/storage/presence.py
+++ b/synapse/storage/presence.py
@@ -98,7 +98,7 @@ class PresenceStore(SQLBaseStore):
             updatevalues={"accepted": True},
             desc="set_presence_list_accepted",
         )
-        self.get_presence_list_accepted.invalidate(observer_localpart)
+        self.get_presence_list_accepted.invalidate((observer_localpart,))
         defer.returnValue(result)
 
     def get_presence_list(self, observer_localpart, accepted=None):
@@ -133,4 +133,4 @@ class PresenceStore(SQLBaseStore):
                        "observed_user_id": observed_userid},
             desc="del_presence_list",
         )
-        self.get_presence_list_accepted.invalidate(observer_localpart)
+        self.get_presence_list_accepted.invalidate((observer_localpart,))
diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index a220f3632..9b88ca7b3 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -151,11 +151,11 @@ class PushRuleStore(SQLBaseStore):
             txn.execute(sql, (user_name, priority_class, new_rule_priority))
 
         txn.call_after(
-            self.get_push_rules_for_user.invalidate, user_name
+            self.get_push_rules_for_user.invalidate, (user_name,)
         )
 
         txn.call_after(
-            self.get_push_rules_enabled_for_user.invalidate, user_name
+            self.get_push_rules_enabled_for_user.invalidate, (user_name,)
         )
 
         self._simple_insert_txn(
@@ -187,10 +187,10 @@ class PushRuleStore(SQLBaseStore):
         new_rule['priority'] = new_prio
 
         txn.call_after(
-            self.get_push_rules_for_user.invalidate, user_name
+            self.get_push_rules_for_user.invalidate, (user_name,)
         )
         txn.call_after(
-            self.get_push_rules_enabled_for_user.invalidate, user_name
+            self.get_push_rules_enabled_for_user.invalidate, (user_name,)
         )
 
         self._simple_insert_txn(
@@ -216,8 +216,8 @@ class PushRuleStore(SQLBaseStore):
             desc="delete_push_rule",
         )
 
-        self.get_push_rules_for_user.invalidate(user_name)
-        self.get_push_rules_enabled_for_user.invalidate(user_name)
+        self.get_push_rules_for_user.invalidate((user_name,))
+        self.get_push_rules_enabled_for_user.invalidate((user_name,))
 
     @defer.inlineCallbacks
     def set_push_rule_enabled(self, user_name, rule_id, enabled):
@@ -238,10 +238,10 @@ class PushRuleStore(SQLBaseStore):
             {'id': new_id},
         )
         txn.call_after(
-            self.get_push_rules_for_user.invalidate, user_name
+            self.get_push_rules_for_user.invalidate, (user_name,)
         )
         txn.call_after(
-            self.get_push_rules_enabled_for_user.invalidate, user_name
+            self.get_push_rules_enabled_for_user.invalidate, (user_name,)
         )
 
 
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 90e2606be..4eaa088b3 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -131,7 +131,7 @@ class RegistrationStore(SQLBaseStore):
             user_id
         )
         for r in rows:
-            self.get_user_by_token.invalidate(r)
+            self.get_user_by_token.invalidate((r,))
 
     @cached()
     def get_user_by_token(self, token):
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 55dd3f6cf..9f14f38f2 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -54,9 +54,9 @@ class RoomMemberStore(SQLBaseStore):
         )
 
         for event in events:
-            txn.call_after(self.get_rooms_for_user.invalidate, event.state_key)
-            txn.call_after(self.get_joined_hosts_for_room.invalidate, event.room_id)
-            txn.call_after(self.get_users_in_room.invalidate, event.room_id)
+            txn.call_after(self.get_rooms_for_user.invalidate, (event.state_key,))
+            txn.call_after(self.get_joined_hosts_for_room.invalidate, (event.room_id,))
+            txn.call_after(self.get_users_in_room.invalidate, (event.room_id,))
 
     def get_room_member(self, user_id, room_id):
         """Retrieve the current state of a room member.
diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py
index 8fa305d18..abee2f631 100644
--- a/tests/storage/test__base.py
+++ b/tests/storage/test__base.py
@@ -42,12 +42,12 @@ class CacheTestCase(unittest.TestCase):
         self.assertEquals(self.cache.get("foo"), 123)
 
     def test_invalidate(self):
-        self.cache.prefill("foo", 123)
-        self.cache.invalidate("foo")
+        self.cache.prefill(("foo",), 123)
+        self.cache.invalidate(("foo",))
 
         failed = False
         try:
-            self.cache.get("foo")
+            self.cache.get(("foo",))
         except KeyError:
             failed = True
 
@@ -141,7 +141,7 @@ class CacheDecoratorTestCase(unittest.TestCase):
 
         self.assertEquals(callcount[0], 1)
 
-        a.func.invalidate("foo")
+        a.func.invalidate(("foo",))
 
         yield a.func("foo")
 
@@ -153,7 +153,7 @@ class CacheDecoratorTestCase(unittest.TestCase):
             def func(self, key):
                 return key
 
-        A().func.invalidate("what")
+        A().func.invalidate(("what",))
 
     @defer.inlineCallbacks
     def test_max_entries(self):
@@ -193,7 +193,7 @@ class CacheDecoratorTestCase(unittest.TestCase):
 
         a = A()
 
-        a.func.prefill("foo", ObservableDeferred(d))
+        a.func.prefill(("foo",), ObservableDeferred(d))
 
         self.assertEquals(a.func("foo").result, d.result)
         self.assertEquals(callcount[0], 0)

From 3213ff630ca6f7f624d65dbab55e2c4c65fe80ce Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 7 Aug 2015 19:13:45 +0100
Subject: [PATCH 21/38] Remove unnecessary cache

---
 synapse/storage/state.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 55c6d5289..7ce51b9bd 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import SQLBaseStore, cached, cachedInlineCallbacks
+from ._base import SQLBaseStore, cachedInlineCallbacks
 
 from twisted.internet import defer
 
@@ -91,7 +91,6 @@ class StateStore(SQLBaseStore):
 
         defer.returnValue(dict(state_list))
 
-    @cached(num_args=1)
     def _fetch_events_for_group(self, key, events):
         return self._get_events(
             events, get_prev_content=False

From 62126c996c5bc8d86d3e15c015a68e8e60622a55 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 7 Aug 2015 19:17:58 +0100
Subject: [PATCH 22/38] Propogate stale cache errors to calling functions

---
 synapse/storage/_base.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 0872a438f..524a00315 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -183,8 +183,8 @@ class CacheDescriptor(object):
             try:
                 cached_result_d = cache.get(*keyargs)
 
+                observed = cached_result_d.observe()
                 if DEBUG_CACHES:
-
                     @defer.inlineCallbacks
                     def check_result(cached_result):
                         actual_result = yield self.function_to_call(obj, *args, **kwargs)
@@ -195,9 +195,10 @@ class CacheDescriptor(object):
                                 cached_result, actual_result,
                             )
                             raise ValueError("Stale cache entry")
-                    cached_result_d.observe().addCallback(check_result)
+                        defer.returnValue(cached_result)
+                    observed.addCallback(check_result)
 
-                return cached_result_d.observe()
+                return observed
             except KeyError:
                 # Get the sequence number of the cache before reading from the
                 # database so that we can tell if the cache is invalidated

From 9c5385b53ad6d8ac21227dbc3f150c7d2511b061 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 7 Aug 2015 19:26:38 +0100
Subject: [PATCH 23/38] s/observed/observer/

---
 synapse/storage/_base.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 524a00315..5997603b3 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -183,7 +183,7 @@ class CacheDescriptor(object):
             try:
                 cached_result_d = cache.get(*keyargs)
 
-                observed = cached_result_d.observe()
+                observer = cached_result_d.observe()
                 if DEBUG_CACHES:
                     @defer.inlineCallbacks
                     def check_result(cached_result):
@@ -196,9 +196,9 @@ class CacheDescriptor(object):
                             )
                             raise ValueError("Stale cache entry")
                         defer.returnValue(cached_result)
-                    observed.addCallback(check_result)
+                    observer.addCallback(check_result)
 
-                return observed
+                return observer
             except KeyError:
                 # Get the sequence number of the cache before reading from the
                 # database so that we can tell if the cache is invalidated

From 2cd6cb9f65e64ea49e0a8ba7c4851c79821243ad Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 10 Aug 2015 10:38:47 +0100
Subject: [PATCH 24/38] Rename keyargs to args in Cache

---
 synapse/storage/_base.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index a088a6175..1e2d43666 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -86,8 +86,8 @@ class Cache(object):
                     "Cache objects can only be accessed from the main thread"
                 )
 
-    def get(self, keyargs, default=_CacheSentinel):
-        val = self.cache.get(keyargs, _CacheSentinel)
+    def get(self, key, default=_CacheSentinel):
+        val = self.cache.get(key, _CacheSentinel)
         if val is not _CacheSentinel:
             cache_counter.inc_hits(self.name)
             return val
@@ -99,29 +99,29 @@ class Cache(object):
         else:
             return default
 
-    def update(self, sequence, keyargs, value):
+    def update(self, sequence, key, value):
         self.check_thread()
         if self.sequence == sequence:
             # Only update the cache if the caches sequence number matches the
             # number that the cache had before the SELECT was started (SYN-369)
-            self.prefill(keyargs, value)
+            self.prefill(key, value)
 
-    def prefill(self, keyargs, value):
+    def prefill(self, key, value):
         if self.max_entries is not None:
             while len(self.cache) >= self.max_entries:
                 self.cache.popitem(last=False)
 
-        self.cache[keyargs] = value
+        self.cache[key] = value
 
-    def invalidate(self, keyargs):
+    def invalidate(self, key):
         self.check_thread()
-        if not isinstance(keyargs, tuple):
+        if not isinstance(key, tuple):
             raise ValueError("keyargs must be a tuple.")
 
         # Increment the sequence number so that any SELECT statements that
         # raced with the INSERT don't update the cache (SYN-369)
         self.sequence += 1
-        self.cache.pop(keyargs, None)
+        self.cache.pop(key, None)
 
     def invalidate_all(self):
         self.check_thread()

From 86eaaa885bb4f3d75990874a76c53b2a0870d639 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 10 Aug 2015 13:44:44 +0100
Subject: [PATCH 25/38] Rename keyargs to args in CacheDescriptor

---
 synapse/storage/_base.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 1e2d43666..e76ee2779 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -180,9 +180,9 @@ class CacheDescriptor(object):
         @functools.wraps(self.orig)
         def wrapped(*args, **kwargs):
             arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs)
-            keyargs = tuple(arg_dict[arg_nm] for arg_nm in self.arg_names)
+            cache_key = tuple(arg_dict[arg_nm] for arg_nm in self.arg_names)
             try:
-                cached_result_d = self.cache.get(keyargs)
+                cached_result_d = self.cache.get(cache_key)
 
                 observer = cached_result_d.observe()
                 if DEBUG_CACHES:
@@ -192,7 +192,7 @@ class CacheDescriptor(object):
                         if actual_result != cached_result:
                             logger.error(
                                 "Stale cache entry %s%r: cached: %r, actual %r",
-                                self.orig.__name__, keyargs,
+                                self.orig.__name__, cache_key,
                                 cached_result, actual_result,
                             )
                             raise ValueError("Stale cache entry")
@@ -212,13 +212,13 @@ class CacheDescriptor(object):
                 )
 
                 def onErr(f):
-                    self.cache.invalidate(keyargs)
+                    self.cache.invalidate(cache_key)
                     return f
 
                 ret.addErrback(onErr)
 
                 ret = ObservableDeferred(ret, consumeErrors=True)
-                self.cache.update(sequence, keyargs, ret)
+                self.cache.update(sequence, cache_key, ret)
 
                 return ret.observe()
 

From 559c51debcf5402cd5999aa1afb854b34e9de363 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Mon, 10 Aug 2015 14:07:08 +0100
Subject: [PATCH 26/38] Use TypeError instead of ValueError and give a nicer
 error mesasge when someone calls Cache.invalidate with the wrong type.

---
 synapse/storage/_base.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index e76ee2779..73eea157a 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -116,7 +116,9 @@ class Cache(object):
     def invalidate(self, key):
         self.check_thread()
         if not isinstance(key, tuple):
-            raise ValueError("keyargs must be a tuple.")
+            raise TypeError(
+                "The cache key must be a tuple not %r" % (type(key),)
+            )
 
         # Increment the sequence number so that any SELECT statements that
         # raced with the INSERT don't update the cache (SYN-369)

From 9dba813234fdeeef5097572dec9c975a4a83a99b Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <daniel@matrix.org>
Date: Tue, 11 Aug 2015 16:34:17 +0100
Subject: [PATCH 27/38] Remove redundant if-guard

The startswith("@") does the job
---
 synapse/api/auth.py | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 487be7ce9..bf02347f6 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -521,23 +521,22 @@ class Auth(object):
 
         # Check state_key
         if hasattr(event, "state_key"):
-            if not event.state_key.startswith("_"):
-                if event.state_key.startswith("@"):
-                    if event.state_key != event.user_id:
+            if event.state_key.startswith("@"):
+                if event.state_key != event.user_id:
+                    raise AuthError(
+                        403,
+                        "You are not allowed to set others state"
+                    )
+                else:
+                    sender_domain = UserID.from_string(
+                        event.user_id
+                    ).domain
+
+                    if sender_domain != event.state_key:
                         raise AuthError(
                             403,
                             "You are not allowed to set others state"
                         )
-                    else:
-                        sender_domain = UserID.from_string(
-                            event.user_id
-                        ).domain
-
-                        if sender_domain != event.state_key:
-                            raise AuthError(
-                                403,
-                                "You are not allowed to set others state"
-                            )
 
         return True
 

From 88e03da39f3746f7d754150964da064772b6ba5f Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <daniel@matrix.org>
Date: Tue, 11 Aug 2015 16:35:28 +0100
Subject: [PATCH 28/38] Minor docs cleanup

---
 synapse/api/auth.py | 9 +++++++--
 synapse/types.py    | 3 ++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index bf02347f6..0eef7442e 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -44,6 +44,11 @@ class Auth(object):
     def check(self, event, auth_events):
         """ Checks if this event is correctly authed.
 
+        Args:
+            event: the event being checked.
+            auth_events (dict: event-key -> event): the existing room state.
+
+
         Returns:
             True if the auth checks pass.
         """
@@ -319,7 +324,7 @@ class Auth(object):
         Returns:
             tuple : of UserID and device string:
                 User ID object of the user making the request
-                Client ID object of the client instance the user is using
+                ClientInfo object of the client instance the user is using
         Raises:
             AuthError if no user by that token exists or the token is invalid.
         """
@@ -352,7 +357,7 @@ class Auth(object):
                 )
                 return
             except KeyError:
-                pass  # normal users won't have this query parameter set
+                pass # normal users won't have the user_id query parameter set.
 
             user_info = yield self.get_user_by_token(access_token)
             user = user_info["user"]
diff --git a/synapse/types.py b/synapse/types.py
index dd1b10d64..e190374cb 100644
--- a/synapse/types.py
+++ b/synapse/types.py
@@ -178,7 +178,7 @@ class RoomStreamToken(namedtuple("_StreamToken", "topological stream")):
 
     Live tokens start with an "s" followed by the "stream_ordering" id of the
     event it comes after. Historic tokens start with a "t" followed by the
-    "topological_ordering" id of the event it comes after, follewed by "-",
+    "topological_ordering" id of the event it comes after, followed by "-",
     followed by the "stream_ordering" id of the event it comes after.
     """
     __slots__ = []
@@ -211,4 +211,5 @@ class RoomStreamToken(namedtuple("_StreamToken", "topological stream")):
             return "s%d" % (self.stream,)
 
 
+# token_id is the primary key ID of the access token, not the access token itself.
 ClientInfo = namedtuple("ClientInfo", ("device_id", "token_id"))

From 45610305eac1d5ce15faf0bf0482442aa896f4a6 Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <daniel@matrix.org>
Date: Tue, 11 Aug 2015 16:43:27 +0100
Subject: [PATCH 29/38] Add missing space because linter

---
 synapse/api/auth.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 0eef7442e..a7f428a96 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -357,7 +357,7 @@ class Auth(object):
                 )
                 return
             except KeyError:
-                pass # normal users won't have the user_id query parameter set.
+                pass  # normal users won't have the user_id query parameter set.
 
             user_info = yield self.get_user_by_token(access_token)
             user = user_info["user"]

From 4ff0228c254e146e37ffd2b1c1df22d1b67353cb Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <daniel@matrix.org>
Date: Tue, 11 Aug 2015 16:54:06 +0100
Subject: [PATCH 30/38] Remove call to recently removed function in mock

---
 tests/test_distributor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_distributor.py b/tests/test_distributor.py
index 6a0095d85..8ed48cfb7 100644
--- a/tests/test_distributor.py
+++ b/tests/test_distributor.py
@@ -73,8 +73,8 @@ class DistributorTestCase(unittest.TestCase):
             yield d
             self.assertTrue(d.called)
 
-            observers[0].assert_called_once("Go")
-            observers[1].assert_called_once("Go")
+            observers[0].assert_called_once_with("Go")
+            observers[1].assert_called_once_with("Go")
 
             self.assertEquals(mock_logger.warning.call_count, 1)
             self.assertIsInstance(mock_logger.warning.call_args[0][0],

From b24d7ebd6e57154d13f3a3a8e1aacc20ef34b417 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 12 Aug 2015 09:39:07 +0100
Subject: [PATCH 31/38] Fix the cleanup script to use the right $DIR

---
 demo/clean.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/demo/clean.sh b/demo/clean.sh
index 31007737f..418ca9457 100755
--- a/demo/clean.sh
+++ b/demo/clean.sh
@@ -12,8 +12,8 @@ if [ -f $PID_FILE ]; then
 fi
 
 for port in 8080 8081 8082; do
-    rm -rf demo/$port
-    rm -rf demo/media_store.$port
+    rm -rf $DIR/$port
+    rm -rf $DIR/media_store.$port
 done
 
 rm -rf $DIR/etc

From 7bbaab94327d806617fc136aab51303baed060cf Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 12 Aug 2015 11:57:37 +0100
Subject: [PATCH 32/38] Fix the --generate-keys option. Make it do the same
 thing as --generate-config does when the config file exists, but without
 printing a warning

---
 synapse/config/_base.py | 83 ++++++++++++++++++-----------------------
 1 file changed, 37 insertions(+), 46 deletions(-)

diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index c408db2b4..73f695995 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -149,6 +149,8 @@ class Config(object):
         )
         config_args, remaining_args = config_parser.parse_known_args(argv)
 
+        generate_keys = config_args.generate_keys
+
         if config_args.generate_config:
             if not config_args.config_path:
                 config_parser.error(
@@ -156,51 +158,40 @@ class Config(object):
                     " generated using \"--generate-config -H SERVER_NAME"
                     " -c CONFIG-FILE\""
                 )
-
-            config_dir_path = os.path.dirname(config_args.config_path[0])
-            config_dir_path = os.path.abspath(config_dir_path)
-
-            server_name = config_args.server_name
-            if not server_name:
-                print "Must specify a server_name to a generate config for."
-                sys.exit(1)
             (config_path,) = config_args.config_path
-            if not os.path.exists(config_dir_path):
-                os.makedirs(config_dir_path)
-            if os.path.exists(config_path):
-                print "Config file %r already exists" % (config_path,)
-                yaml_config = cls.read_config_file(config_path)
-                yaml_name = yaml_config["server_name"]
-                if server_name != yaml_name:
-                    print (
-                        "Config file %r has a different server_name: "
-                        " %r != %r" % (config_path, server_name, yaml_name)
-                    )
+            if not os.path.exists(config_path):
+                config_dir_path = os.path.dirname(config_path)
+                config_dir_path = os.path.abspath(config_dir_path)
+
+                server_name = config_args.server_name
+                if not server_name:
+                    print "Must specify a server_name to a generate config for."
                     sys.exit(1)
-                config_bytes, config = obj.generate_config(
-                    config_dir_path, server_name
-                )
-                config.update(yaml_config)
-                print "Generating any missing keys for %r" % (server_name,)
-                obj.invoke_all("generate_files", config)
-                sys.exit(0)
-            with open(config_path, "wb") as config_file:
-                config_bytes, config = obj.generate_config(
-                    config_dir_path, server_name
-                )
-                obj.invoke_all("generate_files", config)
-                config_file.write(config_bytes)
+                if not os.path.exists(config_dir_path):
+                    os.makedirs(config_dir_path)
+                with open(config_path, "wb") as config_file:
+                    config_bytes, config = obj.generate_config(
+                        config_dir_path, server_name
+                    )
+                    obj.invoke_all("generate_files", config)
+                    config_file.write(config_bytes)
                 print (
-                    "A config file has been generated in %s for server name"
-                    " '%s' with corresponding SSL keys and self-signed"
-                    " certificates. Please review this file and customise it to"
-                    " your needs."
+                    "A config file has been generated in %r for server name"
+                    " %r with corresponding SSL keys and self-signed"
+                    " certificates. Please review this file and customise it"
+                    " to your needs."
                 ) % (config_path, server_name)
-            print (
-                "If this server name is incorrect, you will need to regenerate"
-                " the SSL certificates"
-            )
-            sys.exit(0)
+                print (
+                    "If this server name is incorrect, you will need to"
+                    " regenerate the SSL certificates"
+                )
+                sys.exit(0)
+            else:
+                print (
+                    "Config file %r already exists. Generating any missing key"
+                    " files."
+                ) % (config_path,)
+                generate_keys = True
 
         parser = argparse.ArgumentParser(
             parents=[config_parser],
@@ -218,7 +209,7 @@ class Config(object):
                 " -c CONFIG-FILE\""
             )
 
-        config_dir_path = os.path.dirname(config_args.config_path[0])
+        config_dir_path = os.path.dirname(config_args.config_path[-1])
         config_dir_path = os.path.abspath(config_dir_path)
 
         specified_config = {}
@@ -231,12 +222,12 @@ class Config(object):
         config.pop("log_config")
         config.update(specified_config)
 
+        if generate_keys:
+            obj.invoke_all("generate_files", config)
+            sys.exit(0)
+
         obj.invoke_all("read_config", config)
 
         obj.invoke_all("read_arguments", args)
 
-        if config_args.generate_keys:
-            obj.invoke_all("generate_files", config)
-            sys.exit(0)
-
         return obj

From 2eb91e6694f43854c28fe33326cb2e8be4ac69c5 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 12 Aug 2015 16:53:30 +0100
Subject: [PATCH 33/38] enable registration in the demo servers

---
 demo/start.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/demo/start.sh b/demo/start.sh
index b5dea5e17..572dbfab0 100755
--- a/demo/start.sh
+++ b/demo/start.sh
@@ -23,7 +23,6 @@ for port in 8080 8081 8082; do
     #rm $DIR/etc/$port.config
     python -m synapse.app.homeserver \
         --generate-config \
-        --enable_registration \
         -H "localhost:$https_port" \
         --config-path "$DIR/etc/$port.config" \
 
@@ -36,6 +35,8 @@ for port in 8080 8081 8082; do
         fi
     fi
 
+    perl -p -i -e 's/^enable_registration:.*/enable_registration: true/g' $DIR/etc/$port.config
+
     python -m synapse.app.homeserver \
         --config-path "$DIR/etc/$port.config" \
         -D \

From 21ac8be5f7be79cfffb32ad9fe1bba9515d9fd3e Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 12 Aug 2015 17:25:13 +0100
Subject: [PATCH 34/38] Depend on Twisted>=15.1 rather than pining to a
 particular version

---
 setup.py                       | 2 +-
 synapse/python_dependencies.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index c790484c7..16ccc0f1b 100755
--- a/setup.py
+++ b/setup.py
@@ -48,7 +48,7 @@ setup(
     description="Reference Synapse Home Server",
     install_requires=dependencies['requirements'](include_conditional=True).keys(),
     setup_requires=[
-        "Twisted==15.2.1", # Here to override setuptools_trial's dependency on Twisted>=2.4.0
+        "Twisted>=15.1.0", # Here to override setuptools_trial's dependency on Twisted>=2.4.0
         "setuptools_trial",
         "mock"
     ],
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 534c4a669..fa06480ad 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
 
 REQUIREMENTS = {
     "syutil>=0.0.7": ["syutil>=0.0.7"],
-    "Twisted==15.2.1": ["twisted==15.2.1"],
+    "Twisted>=15.1.0": ["twisted>=15.1.0"],
     "service_identity>=1.0.0": ["service_identity>=1.0.0"],
     "pyopenssl>=0.14": ["OpenSSL>=0.14"],
     "pyyaml": ["yaml"],

From ba5d34a83274127a2c0226059778d226355bdb6c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 13 Aug 2015 11:38:59 +0100
Subject: [PATCH 35/38] Add some metrics about the reactor

---
 synapse/metrics/__init__.py | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index 9233ea3da..e014b415f 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -18,8 +18,12 @@ from __future__ import absolute_import
 
 import logging
 from resource import getrusage, getpagesize, RUSAGE_SELF
+import functools
 import os
 import stat
+import time
+
+from twisted.internet import reactor
 
 from .metric import (
     CounterMetric, CallbackMetric, DistributionMetric, CacheMetric
@@ -144,3 +148,28 @@ def _process_fds():
     return counts
 
 get_metrics_for("process").register_callback("fds", _process_fds, labels=["type"])
+
+reactor_metrics = get_metrics_for("reactor")
+tick_time = reactor_metrics.register_distribution("tick_time")
+pending_calls_metric = reactor_metrics.register_distribution("pending_calls")
+
+
+def runUntilCurrentTimer(func):
+
+    @functools.wraps(func)
+    def f(*args, **kwargs):
+        start = time.time() * 1000
+        pending_calls = len(reactor.getDelayedCalls())
+        ret = func(*args, **kwargs)
+        end = time.time() * 1000
+        tick_time.inc_by(end - start)
+        pending_calls_metric.inc_by(pending_calls)
+        return ret
+
+    return f
+
+
+if hasattr(reactor, "runUntilCurrent"):
+    # runUntilCurrent is called when we have pending calls. It is called once
+    # per iteratation after fd polling.
+    reactor.runUntilCurrent = runUntilCurrentTimer(reactor.runUntilCurrent)

From a6c27de1aa283a9d7a347db53b08367c53a15a24 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 13 Aug 2015 11:41:57 +0100
Subject: [PATCH 36/38] Don't time getDelayedCalls

---
 synapse/metrics/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index e014b415f..0be977299 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -158,8 +158,8 @@ def runUntilCurrentTimer(func):
 
     @functools.wraps(func)
     def f(*args, **kwargs):
-        start = time.time() * 1000
         pending_calls = len(reactor.getDelayedCalls())
+        start = time.time() * 1000
         ret = func(*args, **kwargs)
         end = time.time() * 1000
         tick_time.inc_by(end - start)

From adbd720fab5f6ee42c4e14b06eb6f385bae14dc6 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 13 Aug 2015 11:47:38 +0100
Subject: [PATCH 37/38] PEP8

---
 synapse/http/matrixfederationclient.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 4d74bd5d7..854e17a47 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -142,7 +142,6 @@ class MatrixFederationHttpClient(object):
                             producer
                         )
 
-
                         return self.clock.time_bound_deferred(
                             request_deferred,
                             time_out=timeout/1000. if timeout else 60,

From 7eb4d626bada90834f8cfe464e4424ed8991e590 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Thu, 13 Aug 2015 13:12:33 +0100
Subject: [PATCH 38/38] Add max-line-length to the flake8 section of setup.cfg

---
 setup.cfg | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/setup.cfg b/setup.cfg
index 888ad6ed4..abb649958 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -16,3 +16,6 @@ ignore =
     docs/*
     pylint.cfg
     tox.ini
+
+[flake8]
+max-line-length = 90