Merge branch 'develop' of github.com:matrix-org/synapse into timeout-federation-requests

CHANGES.rst

@@ -2,6 +2,8 @@ Changes in develop
 ==================
 
  * pydenticon support -- adds dep on pydenticon
+ * pylru
+ * simplejson
 
 Changes in synapse 0.6.1 (2015-01-07)
 =====================================

synapse/config/_base.py

@@ -27,6 +27,16 @@ class Config(object):
     def __init__(self, args):
         pass
 
+    @staticmethod
+    def parse_size(string):
+        sizes = {"K": 1024, "M": 1024 * 1024}
+        size = 1
+        suffix = string[-1]
+        if suffix in sizes:
+            string = string[:-1]
+            size = sizes[suffix]
+        return int(string) * size
+
     @staticmethod
     def abspath(file_path):
         return os.path.abspath(file_path) if file_path else file_path

synapse/config/database.py

@@ -24,6 +24,7 @@ class DatabaseConfig(Config):
             self.database_path = ":memory:"
         else:
             self.database_path = self.abspath(args.database_path)
+        self.event_cache_size = self.parse_size(args.event_cache_size)
 
     @classmethod
     def add_arguments(cls, parser):
@@ -33,6 +34,10 @@ class DatabaseConfig(Config):
             "-d", "--database-path", default="homeserver.db",
             help="The database name."
         )
+        db_group.add_argument(
+            "--event-cache-size", default="100K",
+            help="Number of events to cache in memory."
+        )
 
     @classmethod
     def generate_config(cls, args, config_dir_path):

synapse/http/server.py

@@ -113,6 +113,7 @@ class JsonResource(HttpServer, resource.Resource):
             path.
         """
         code = None
+        start = self.clock.time_msec()
         try:
             # Just say yes to OPTIONS.
             if request.method == "OPTIONS":
@@ -137,8 +138,6 @@ class JsonResource(HttpServer, resource.Resource):
                         request.method, request.path
                     )
 
-                    start = self.clock.time_msec()
-
                     code, response = yield path_entry.callback(
                         request,
                         *args

synapse/storage/__init__.py

@@ -164,6 +164,9 @@ class DataStore(RoomMemberStore, RoomStore,
                            stream_ordering=None, is_new_state=True,
                            current_state=None):
 
+        # Remove the any existing cache entries for the event_id
+        self._get_event_cache.pop(event.event_id)
+
         # We purposefully do this first since if we include a `current_state`
         # key, we *want* to update the `current_state_events` table
         if current_state:
@@ -420,6 +423,8 @@ class DataStore(RoomMemberStore, RoomStore,
         )
 
     def _store_redaction(self, txn, event):
+        # invalidate the cache for the redacted event
+        self._get_event_cache.pop(event.redacts)
         txn.execute(
             "INSERT OR IGNORE INTO redactions "
             "(event_id, redacts) VALUES (?,?)",

synapse/storage/_base.py

@@ -19,6 +19,7 @@ from synapse.events import FrozenEvent
 from synapse.events.utils import prune_event
 from synapse.util.logutils import log_function
 from synapse.util.logcontext import PreserveLoggingContext, LoggingContext
+from synapse.util.lrucache import LruCache
 
 from twisted.internet import defer
 
@@ -128,6 +129,8 @@ class SQLBaseStore(object):
         self._txn_perf_counters = PerformanceCounters()
         self._get_event_counters = PerformanceCounters()
 
+        self._get_event_cache = LruCache(hs.config.event_cache_size)
+
     def start_profiling(self):
         self._previous_loop_ts = self._clock.time_msec()
 
@@ -579,6 +582,19 @@ class SQLBaseStore(object):
 
     def _get_event_txn(self, txn, event_id, check_redacted=True,
                        get_prev_content=False, allow_rejected=False):
+
+        start_time = time.time() * 1000
+        update_counter = self._get_event_counters.update
+
+        try:
+            cache = self._get_event_cache.setdefault(event_id, {})
+            # Separate cache entries for each way to invoke _get_event_txn
+            return cache[(check_redacted, get_prev_content, allow_rejected)]
+        except KeyError:
+            pass
+        finally:
+            start_time = update_counter("event_cache", start_time)
+
         sql = (
             "SELECT e.internal_metadata, e.json, r.event_id, rej.reason "
             "FROM event_json as e "
@@ -588,8 +604,6 @@ class SQLBaseStore(object):
             "LIMIT 1 "
         )
 
-        start_time = time.time() * 1000
-
         txn.execute(sql, (event_id,))
 
         res = txn.fetchone()
@@ -599,14 +613,16 @@ class SQLBaseStore(object):
 
         internal_metadata, js, redacted, rejected_reason = res
 
-        self._get_event_counters.update("select_event", start_time)
+        start_time = update_counter("select_event", start_time)
 
         if allow_rejected or not rejected_reason:
-            return self._get_event_from_row_txn(
+            result = self._get_event_from_row_txn(
                 txn, internal_metadata, js, redacted,
                 check_redacted=check_redacted,
                 get_prev_content=get_prev_content,
             )
+            cache[(check_redacted, get_prev_content, allow_rejected)] = result
+            return result
         else:
             return None
 

synapse/util/lrucache.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class LruCache(object):
+    """Least-recently-used cache."""
+    # TODO(mjark) Add hit/miss counters
+    # TODO(mjark) Add mutex for linked list for thread safety.
+    def __init__(self, max_size):
+        cache = {}
+        list_root = []
+        list_root[:] = [list_root, list_root, None, None]
+
+        PREV, NEXT, KEY, VALUE = 0, 1, 2, 3
+
+        def add_node(key, value):
+            prev_node = list_root
+            next_node = prev_node[NEXT]
+            node = [prev_node, next_node, key, value]
+            prev_node[NEXT] = node
+            next_node[PREV] = node
+            cache[key] = node
+
+        def move_node_to_front(node):
+            prev_node = node[PREV]
+            next_node = node[NEXT]
+            prev_node[NEXT] = next_node
+            next_node[PREV] = prev_node
+            prev_node = list_root
+            next_node = prev_node[NEXT]
+            node[PREV] = prev_node
+            node[NEXT] = next_node
+            prev_node[NEXT] = node
+            next_node[PREV] = node
+
+        def delete_node(node):
+            prev_node = node[PREV]
+            next_node = node[NEXT]
+            prev_node[NEXT] = next_node
+            next_node[PREV] = prev_node
+            cache.pop(node[KEY], None)
+
+        def cache_get(key, default=None):
+            node = cache.get(key, None)
+            if node is not None:
+                move_node_to_front(node)
+                return node[VALUE]
+            else:
+                return default
+
+        def cache_set(key, value):
+            node = cache.get(key, None)
+            if node is not None:
+                move_node_to_front(node)
+                node[VALUE] = value
+            else:
+                add_node(key, value)
+                if len(cache) > max_size:
+                    delete_node(list_root[PREV])
+
+        def cache_set_default(key, value):
+            node = cache.get(key, None)
+            if node is not None:
+                return node[VALUE]
+            else:
+                add_node(key, value)
+                if len(cache) > max_size:
+                    delete_node(list_root[PREV])
+                return value
+
+        def cache_pop(key, default=None):
+            node = cache.get(key, None)
+            if node:
+                delete_node(node)
+                return node[VALUE]
+            else:
+                return default
+
+        self.sentinel = object()
+        self.get = cache_get
+        self.set = cache_set
+        self.setdefault = cache_set_default
+        self.pop = cache_pop
+
+    def __getitem__(self, key):
+        result = self.get(key, self.sentinel)
+        if result is self.sentinel:
+            raise KeyError()
+        else:
+            return result
+
+    def __setitem__(self, key, value):
+        self.set(key, value)
+
+    def __delitem__(self, key, value):
+        result = self.pop(key, self.sentinel)
+        if result is self.sentinel:
+            raise KeyError()

tests/api/test_filtering.py

@@ -19,11 +19,9 @@ from twisted.internet import defer
 
 from mock import Mock, NonCallableMock
 from tests.utils import (
-    MockHttpResource, MockClock, DeferredMockCallable, SQLiteMemoryDbPool,
+    MockHttpResource, DeferredMockCallable, setup_test_homeserver
-    MockKey
 )
 
-from synapse.server import HomeServer
 from synapse.types import UserID
 from synapse.api.filtering import Filter
 
@@ -34,22 +32,14 @@ class FilteringTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        self.mock_config = NonCallableMock()
-        self.mock_config.signing_key = [MockKey()]
-
         self.mock_federation_resource = MockHttpResource()
 
         self.mock_http_client = Mock(spec=[])
         self.mock_http_client.put_json = DeferredMockCallable()
 
-        hs = HomeServer("test",
+        hs = yield setup_test_homeserver(
-            db_pool=db_pool,
             handlers=None,
             http_client=self.mock_http_client,
-            config=self.mock_config,
             keyring=Mock(),
         )
 

tests/federation/test_federation.py

@@ -19,9 +19,8 @@ from tests import unittest
 # python imports
 from mock import Mock, ANY
 
-from ..utils import MockHttpResource, MockClock, MockKey
+from ..utils import MockHttpResource, MockClock, setup_test_homeserver
 
-from synapse.server import HomeServer
 from synapse.federation import initialize_http_replication
 from synapse.events import FrozenEvent
 
@@ -40,6 +39,7 @@ def make_pdu(prev_pdus=[], **kwargs):
 
 
 class FederationTestCase(unittest.TestCase):
+    @defer.inlineCallbacks
     def setUp(self):
         self.mock_resource = MockHttpResource()
         self.mock_http_client = Mock(spec=[
@@ -61,17 +61,12 @@ class FederationTestCase(unittest.TestCase):
             defer.succeed(DestinationsTable.EntryType("", 0, 0))
         )
         self.mock_persistence.get_auth_chain.return_value = []
-        self.mock_config = Mock()
-        self.mock_config.signing_key = [MockKey()]
         self.clock = MockClock()
-        hs = HomeServer(
+        hs = yield setup_test_homeserver(
-            "test",
             resource_for_federation=self.mock_resource,
             http_client=self.mock_http_client,
-            db_pool=None,
             datastore=self.mock_persistence,
             clock=self.clock,
-            config=self.mock_config,
             keyring=Mock(),
         )
         self.federation = initialize_http_replication(hs)

tests/handlers/test_directory.py

@@ -19,11 +19,10 @@ from twisted.internet import defer
 
 from mock import Mock
 
-from synapse.server import HomeServer
 from synapse.handlers.directory import DirectoryHandler
 from synapse.types import RoomAlias
 
-from tests.utils import SQLiteMemoryDbPool, MockKey
+from tests.utils import setup_test_homeserver
 
 
 class DirectoryHandlers(object):
@@ -46,19 +45,10 @@ class DirectoryTestCase(unittest.TestCase):
             self.query_handlers[query_type] = handler
         self.mock_federation.register_query_handler = register_query_handler
 
-        db_pool = SQLiteMemoryDbPool()
+        hs = yield setup_test_homeserver(
-        yield db_pool.prepare()
-
-        self.mock_config = Mock()
-        self.mock_config.signing_key = [MockKey()]
-
-        hs = HomeServer(
-            "test",
-            db_pool=db_pool,
             http_client=None,
             resource_for_federation=Mock(),
             replication_layer=self.mock_federation,
-            config=self.mock_config,
         )
         hs.handlers = DirectoryHandlers(hs)
 

tests/handlers/test_federation.py

@@ -19,20 +19,17 @@ from tests import unittest
 from synapse.api.constants import EventTypes
 from synapse.events import FrozenEvent
 from synapse.handlers.federation import FederationHandler
-from synapse.server import HomeServer
 
 from mock import NonCallableMock, ANY, Mock
 
-from ..utils import MockKey
+from ..utils import setup_test_homeserver
 
 
 class FederationTestCase(unittest.TestCase):
 
+    @defer.inlineCallbacks
     def setUp(self):
 
-        self.mock_config = NonCallableMock()
-        self.mock_config.signing_key = [MockKey()]
-
         self.state_handler = NonCallableMock(spec_set=[
             "compute_event_context",
         ])
@@ -43,9 +40,8 @@ class FederationTestCase(unittest.TestCase):
         ])
 
         self.hostname = "test"
-        hs = HomeServer(
+        hs = yield setup_test_homeserver(
             self.hostname,
-            db_pool=None,
             datastore=NonCallableMock(spec_set=[
                 "persist_event",
                 "store_room",
@@ -61,7 +57,6 @@ class FederationTestCase(unittest.TestCase):
                 "room_member_handler",
                 "federation_handler",
             ]),
-            config=self.mock_config,
             auth=self.auth,
             state_handler=self.state_handler,
             keyring=Mock(),

tests/handlers/test_presence.py

@@ -21,11 +21,9 @@ from mock import Mock, call, ANY, NonCallableMock
 import json
 
 from tests.utils import (
-    MockHttpResource, MockClock, DeferredMockCallable, SQLiteMemoryDbPool,
+    MockHttpResource, MockClock, DeferredMockCallable, setup_test_homeserver
-    MockKey
 )
 
-from synapse.server import HomeServer
 from synapse.api.constants import PresenceState
 from synapse.api.errors import SynapseError
 from synapse.handlers.presence import PresenceHandler, UserPresenceCache
@@ -66,30 +64,20 @@ class PresenceTestCase(unittest.TestCase):
     def setUp(self):
         self.clock = MockClock()
 
-        self.mock_config = NonCallableMock()
-        self.mock_config.signing_key = [MockKey()]
-
         self.mock_federation_resource = MockHttpResource()
 
         self.mock_http_client = Mock(spec=[])
         self.mock_http_client.put_json = DeferredMockCallable()
 
-        db_pool = None
         hs_kwargs = {}
-
         if hasattr(self, "make_datastore_mock"):
             hs_kwargs["datastore"] = self.make_datastore_mock()
-        else:
-            db_pool = SQLiteMemoryDbPool()
-            yield db_pool.prepare()
 
-        hs = HomeServer("test",
+        hs = yield setup_test_homeserver(
             clock=self.clock,
-            db_pool=db_pool,
             handlers=None,
             resource_for_federation=self.mock_federation_resource,
             http_client=self.mock_http_client,
-            config=self.mock_config,
             keyring=Mock(),
             **hs_kwargs
         )

tests/handlers/test_presencelike.py

@@ -21,9 +21,8 @@ from twisted.internet import defer
 
 from mock import Mock, call, ANY, NonCallableMock
 
-from ..utils import MockClock, MockKey
+from ..utils import MockClock, setup_test_homeserver
 
-from synapse.server import HomeServer
 from synapse.api.constants import PresenceState
 from synapse.handlers.presence import PresenceHandler
 from synapse.handlers.profile import ProfileHandler
@@ -57,29 +56,23 @@ class PresenceAndProfileHandlers(object):
 
 class PresenceProfilelikeDataTestCase(unittest.TestCase):
 
+    @defer.inlineCallbacks
     def setUp(self):
-        self.mock_config = Mock()
+        hs = yield setup_test_homeserver(
-        self.mock_config.signing_key = [MockKey()]
+            clock=MockClock(),
-
+            datastore=Mock(spec=[
-        hs = HomeServer("test",
+                "set_presence_state",
-                clock=MockClock(),
+                "is_presence_visible",
-                db_pool=None,
+                "set_profile_displayname",
-                datastore=Mock(spec=[
+                "get_rooms_for_user_where_membership_is",
-                    "set_presence_state",
+            ]),
-                    "is_presence_visible",
+            handlers=None,
-
+            resource_for_federation=Mock(),
-                    "set_profile_displayname",
+            http_client=None,
-
+            replication_layer=MockReplication(),
-                    "get_rooms_for_user_where_membership_is",
+            ratelimiter=NonCallableMock(spec_set=[
-                ]),
-                handlers=None,
-                resource_for_federation=Mock(),
-                http_client=None,
-                replication_layer=MockReplication(),
-                ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
-                ]),
+            ]),
-                config=self.mock_config
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)

tests/handlers/test_profile.py

@@ -20,11 +20,10 @@ from twisted.internet import defer
 from mock import Mock, NonCallableMock
 
 from synapse.api.errors import AuthError
-from synapse.server import HomeServer
 from synapse.handlers.profile import ProfileHandler
 from synapse.types import UserID
 
-from tests.utils import SQLiteMemoryDbPool, MockKey
+from tests.utils import setup_test_homeserver
 
 
 class ProfileHandlers(object):
@@ -46,23 +45,15 @@ class ProfileTestCase(unittest.TestCase):
             self.query_handlers[query_type] = handler
         self.mock_federation.register_query_handler = register_query_handler
 
-        db_pool = SQLiteMemoryDbPool()
+        hs = yield setup_test_homeserver(
-        yield db_pool.prepare()
+            http_client=None,
-
+            handlers=None,
-        self.mock_config = Mock()
+            resource_for_federation=Mock(),
-        self.mock_config.signing_key = [MockKey()]
+            replication_layer=self.mock_federation,
-
+            ratelimiter=NonCallableMock(spec_set=[
-        hs = HomeServer("test",
+                "send_message",
-                db_pool=db_pool,
+            ])
-                http_client=None,
+        )
-                handlers=None,
-                resource_for_federation=Mock(),
-                replication_layer=self.mock_federation,
-                config=self.mock_config,
-                ratelimiter=NonCallableMock(spec_set=[
-                    "send_message",
-                ])
-            )
 
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)

tests/handlers/test_room.py

@@ -20,23 +20,19 @@ from .. import unittest
 from synapse.api.constants import EventTypes, Membership
 from synapse.handlers.room import RoomMemberHandler, RoomCreationHandler
 from synapse.handlers.profile import ProfileHandler
-from synapse.server import HomeServer
 from synapse.types import UserID
-from ..utils import MockKey
+from ..utils import setup_test_homeserver
 
 from mock import Mock, NonCallableMock
 
 
 class RoomMemberHandlerTestCase(unittest.TestCase):
 
+    @defer.inlineCallbacks
     def setUp(self):
-        self.mock_config = NonCallableMock()
-        self.mock_config.signing_key = [MockKey()]
-
         self.hostname = "red"
-        hs = HomeServer(
+        hs = yield setup_test_homeserver(
             self.hostname,
-            db_pool=None,
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
@@ -64,7 +60,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
                 "compute_event_context",
                 "get_current_state",
             ]),
-            config=self.mock_config,
         )
 
         self.federation = NonCallableMock(spec_set=[
@@ -319,15 +314,12 @@ class RoomMemberHandlerTestCase(unittest.TestCase):
 
 class RoomCreationTest(unittest.TestCase):
 
+    @defer.inlineCallbacks
     def setUp(self):
         self.hostname = "red"
 
-        self.mock_config = NonCallableMock()
+        hs = yield setup_test_homeserver(
-        self.mock_config.signing_key = [MockKey()]
-
-        hs = HomeServer(
             self.hostname,
-            db_pool=None,
             datastore=NonCallableMock(spec_set=[
                 "store_room",
                 "snapshot_room",
@@ -344,7 +336,6 @@ class RoomCreationTest(unittest.TestCase):
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=self.mock_config,
         )
 
         self.federation = NonCallableMock(spec_set=[

tests/handlers/test_typing.py

@@ -20,10 +20,11 @@ from twisted.internet import defer
 from mock import Mock, call, ANY
 import json
 
-from ..utils import MockHttpResource, MockClock, DeferredMockCallable, MockKey
+from ..utils import (
+    MockHttpResource, MockClock, DeferredMockCallable, setup_test_homeserver
+)
 
 from synapse.api.errors import AuthError
-from synapse.server import HomeServer
 from synapse.handlers.typing import TypingNotificationHandler
 
 from synapse.storage.transactions import DestinationsTable
@@ -56,6 +57,7 @@ class JustTypingNotificationHandlers(object):
 
 class TypingNotificationsTestCase(unittest.TestCase):
     """Tests typing notifications to rooms."""
+    @defer.inlineCallbacks
     def setUp(self):
         self.clock = MockClock()
 
@@ -64,34 +66,29 @@ class TypingNotificationsTestCase(unittest.TestCase):
 
         self.mock_federation_resource = MockHttpResource()
 
-        self.mock_config = Mock()
-        self.mock_config.signing_key = [MockKey()]
-
         mock_notifier = Mock(spec=["on_new_user_event"])
         self.on_new_user_event = mock_notifier.on_new_user_event
 
         self.auth = Mock(spec=[])
 
-        hs = HomeServer("test",
+        hs = yield setup_test_homeserver(
-                auth=self.auth,
+            auth=self.auth,
-                clock=self.clock,
+            clock=self.clock,
-                db_pool=None,
+            datastore=Mock(spec=[
-                datastore=Mock(spec=[
+                # Bits that Federation needs
-                    # Bits that Federation needs
+                "prep_send_transaction",
-                    "prep_send_transaction",
+                "delivered_txn",
-                    "delivered_txn",
+                "get_received_txn_response",
-                    "get_received_txn_response",
+                "set_received_txn_response",
-                    "set_received_txn_response",
+                "get_destination_retry_timings",
-                    "get_destination_retry_timings",
+            ]),
-                ]),
+            handlers=None,
-                handlers=None,
+            notifier=mock_notifier,
-                notifier=mock_notifier,
+            resource_for_client=Mock(),
-                resource_for_client=Mock(),
+            resource_for_federation=self.mock_federation_resource,
-                resource_for_federation=self.mock_federation_resource,
+            http_client=self.mock_http_client,
-                http_client=self.mock_http_client,
+            keyring=Mock(),
-                config=self.mock_config,
+        )
-                keyring=Mock(),
-            )
         hs.handlers = JustTypingNotificationHandlers(hs)
 
         self.handler = hs.get_handlers().typing_notification_handler

tests/rest/client/v1/test_events.py

@@ -23,9 +23,8 @@ import synapse.rest.client.v1.events
 import synapse.rest.client.v1.register
 import synapse.rest.client.v1.room
 
-from synapse.server import HomeServer
 
-from ....utils import MockHttpResource, SQLiteMemoryDbPool, MockKey
+from ....utils import MockHttpResource, setup_test_homeserver
 from .utils import RestTestCase
 
 from mock import Mock, NonCallableMock
@@ -113,15 +112,7 @@ class EventStreamPermissionsTestCase(RestTestCase):
     def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
 
-        self.mock_config = NonCallableMock()
+        hs = yield setup_test_homeserver(
-        self.mock_config.signing_key = [MockKey()]
-
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer(
-            "test",
-            db_pool=db_pool,
             http_client=None,
             replication_layer=Mock(),
             clock=Mock(spec=[
@@ -133,7 +124,6 @@ class EventStreamPermissionsTestCase(RestTestCase):
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)

tests/rest/client/v1/test_presence.py

@@ -20,11 +20,10 @@ from twisted.internet import defer
 
 from mock import Mock
 
-from ....utils import MockHttpResource, MockKey
+from ....utils import MockHttpResource, setup_test_homeserver
 
 from synapse.api.constants import PresenceState
 from synapse.handlers.presence import PresenceHandler
-from synapse.server import HomeServer
 from synapse.rest.client.v1 import presence
 from synapse.rest.client.v1 import events
 from synapse.types import UserID
@@ -46,12 +45,10 @@ class JustPresenceHandlers(object):
 
 class PresenceStateTestCase(unittest.TestCase):
 
+    @defer.inlineCallbacks
     def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
-        self.mock_config = Mock()
+        hs = yield setup_test_homeserver(
-        self.mock_config.signing_key = [MockKey()]
-        hs = HomeServer("test",
-            db_pool=None,
             datastore=Mock(spec=[
                 "get_presence_state",
                 "set_presence_state",
@@ -60,7 +57,6 @@ class PresenceStateTestCase(unittest.TestCase):
             http_client=None,
             resource_for_client=self.mock_resource,
             resource_for_federation=self.mock_resource,
-            config=self.mock_config,
         )
         hs.handlers = JustPresenceHandlers(hs)
 
@@ -128,13 +124,11 @@ class PresenceStateTestCase(unittest.TestCase):
 
 class PresenceListTestCase(unittest.TestCase):
 
+    @defer.inlineCallbacks
     def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
-        self.mock_config = Mock()
-        self.mock_config.signing_key = [MockKey()]
 
-        hs = HomeServer("test",
+        hs = yield setup_test_homeserver(
-            db_pool=None,
             datastore=Mock(spec=[
                 "has_presence_state",
                 "get_presence_state",
@@ -149,7 +143,6 @@ class PresenceListTestCase(unittest.TestCase):
             http_client=None,
             resource_for_client=self.mock_resource,
             resource_for_federation=self.mock_resource,
-            config=self.mock_config,
         )
         hs.handlers = JustPresenceHandlers(hs)
 
@@ -242,12 +235,10 @@ class PresenceListTestCase(unittest.TestCase):
 
 
 class PresenceEventStreamTestCase(unittest.TestCase):
+    @defer.inlineCallbacks
     def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
 
-        self.mock_config = Mock()
-        self.mock_config.signing_key = [MockKey()]
-
         # HIDEOUS HACKERY
         # TODO(paul): This should be injected in via the HomeServer DI system
         from synapse.streams.events import (
@@ -264,8 +255,7 @@ class PresenceEventStreamTestCase(unittest.TestCase):
         }
         EventSources.SOURCE_TYPES["presence"] = PresenceEventSource
 
-        hs = HomeServer("test",
+        hs = yield setup_test_homeserver(
-            db_pool=None,
             http_client=None,
             resource_for_client=self.mock_resource,
             resource_for_federation=self.mock_resource,
@@ -278,7 +268,6 @@ class PresenceEventStreamTestCase(unittest.TestCase):
                 "cancel_call_later",
                 "time_msec",
             ]),
-            config=self.mock_config,
         )
 
         hs.get_clock().time_msec.return_value = 1000000

tests/rest/client/v1/test_profile.py

@@ -20,10 +20,9 @@ from twisted.internet import defer
 
 from mock import Mock, NonCallableMock
 
-from ....utils import MockHttpResource, MockKey
+from ....utils import MockHttpResource, setup_test_homeserver
 
 from synapse.api.errors import SynapseError, AuthError
-from synapse.server import HomeServer
 from synapse.types import UserID
 
 from synapse.rest.client.v1 import profile
@@ -35,6 +34,7 @@ PATH_PREFIX = "/_matrix/client/api/v1"
 class ProfileTestCase(unittest.TestCase):
     """ Tests profile management. """
 
+    @defer.inlineCallbacks
     def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
         self.mock_handler = Mock(spec=[
@@ -44,17 +44,12 @@ class ProfileTestCase(unittest.TestCase):
             "set_avatar_url",
         ])
 
-        self.mock_config = NonCallableMock()
+        hs = yield setup_test_homeserver(
-        self.mock_config.signing_key = [MockKey()]
+            "test",
-
-        hs = HomeServer("test",
-            db_pool=None,
             http_client=None,
             resource_for_client=self.mock_resource,
             federation=Mock(),
             replication_layer=Mock(),
-            datastore=None,
-            config=self.mock_config,
         )
 
         def _get_user_by_req(request=None):

tests/rest/client/v1/test_rooms.py

@@ -21,13 +21,12 @@ from twisted.internet import defer
 import synapse.rest.client.v1.room
 from synapse.api.constants import Membership
 
-from synapse.server import HomeServer
 from synapse.types import UserID
 
 import json
 import urllib
 
-from ....utils import MockHttpResource, SQLiteMemoryDbPool, MockKey
+from ....utils import MockHttpResource, setup_test_homeserver
 from .utils import RestTestCase
 
 from mock import Mock, NonCallableMock
@@ -44,21 +43,11 @@ class RoomPermissionsTestCase(RestTestCase):
     def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
 
-        self.mock_config = NonCallableMock()
+        hs = yield setup_test_homeserver(
-        self.mock_config.signing_key = [MockKey()]
-
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer(
             "red",
-            db_pool=db_pool,
             http_client=None,
             replication_layer=Mock(),
-            ratelimiter=NonCallableMock(spec_set=[
+            ratelimiter=NonCallableMock(spec_set=["send_message"]),
-                "send_message",
-            ]),
-            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -439,21 +428,11 @@ class RoomsMemberListTestCase(RestTestCase):
     def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
 
-        self.mock_config = NonCallableMock()
+        hs = yield setup_test_homeserver(
-        self.mock_config.signing_key = [MockKey()]
-
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer(
             "red",
-            db_pool=db_pool,
             http_client=None,
             replication_layer=Mock(),
-            ratelimiter=NonCallableMock(spec_set=[
+            ratelimiter=NonCallableMock(spec_set=["send_message"]),
-                "send_message",
-            ]),
-            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -531,21 +510,11 @@ class RoomsCreateTestCase(RestTestCase):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
         self.auth_user_id = self.user_id
 
-        self.mock_config = NonCallableMock()
+        hs = yield setup_test_homeserver(
-        self.mock_config.signing_key = [MockKey()]
-
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer(
             "red",
-            db_pool=db_pool,
             http_client=None,
             replication_layer=Mock(),
-            ratelimiter=NonCallableMock(spec_set=[
+            ratelimiter=NonCallableMock(spec_set=["send_message"]),
-                "send_message",
-            ]),
-            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -634,21 +603,11 @@ class RoomTopicTestCase(RestTestCase):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
         self.auth_user_id = self.user_id
 
-        self.mock_config = NonCallableMock()
+        hs = yield setup_test_homeserver(
-        self.mock_config.signing_key = [MockKey()]
-
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer(
             "red",
-            db_pool=db_pool,
             http_client=None,
             replication_layer=Mock(),
-            ratelimiter=NonCallableMock(spec_set=[
+            ratelimiter=NonCallableMock(spec_set=["send_message"]),
-                "send_message",
-            ]),
-            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -751,21 +710,11 @@ class RoomMemberStateTestCase(RestTestCase):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
         self.auth_user_id = self.user_id
 
-        self.mock_config = NonCallableMock()
+        hs = yield setup_test_homeserver(
-        self.mock_config.signing_key = [MockKey()]
-
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer(
             "red",
-            db_pool=db_pool,
             http_client=None,
             replication_layer=Mock(),
-            ratelimiter=NonCallableMock(spec_set=[
+            ratelimiter=NonCallableMock(spec_set=["send_message"]),
-                "send_message",
-            ]),
-            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -888,21 +837,11 @@ class RoomMessagesTestCase(RestTestCase):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
         self.auth_user_id = self.user_id
 
-        self.mock_config = NonCallableMock()
+        hs = yield setup_test_homeserver(
-        self.mock_config.signing_key = [MockKey()]
-
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer(
             "red",
-            db_pool=db_pool,
             http_client=None,
             replication_layer=Mock(),
-            ratelimiter=NonCallableMock(spec_set=[
+            ratelimiter=NonCallableMock(spec_set=["send_message"]),
-                "send_message",
-            ]),
-            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)
@@ -993,21 +932,13 @@ class RoomInitialSyncTestCase(RestTestCase):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
         self.auth_user_id = self.user_id
 
-        self.mock_config = NonCallableMock()
+        hs = yield setup_test_homeserver(
-        self.mock_config.signing_key = [MockKey()]
-
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer(
             "red",
-            db_pool=db_pool,
             http_client=None,
             replication_layer=Mock(),
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=self.mock_config,
         )
         self.ratelimiter = hs.get_ratelimiter()
         self.ratelimiter.send_message.return_value = (True, 0)

tests/rest/client/v1/test_typing.py

@@ -19,10 +19,9 @@
 from twisted.internet import defer
 
 import synapse.rest.client.v1.room
-from synapse.server import HomeServer
 from synapse.types import UserID
 
-from ....utils import MockHttpResource, MockClock, SQLiteMemoryDbPool, MockKey
+from ....utils import MockHttpResource, MockClock, setup_test_homeserver
 from .utils import RestTestCase
 
 from mock import Mock, NonCallableMock
@@ -42,22 +41,14 @@ class RoomTypingTestCase(RestTestCase):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
         self.auth_user_id = self.user_id
 
-        self.mock_config = NonCallableMock()
+        hs = yield setup_test_homeserver(
-        self.mock_config.signing_key = [MockKey()]
-
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-
-        hs = HomeServer(
             "red",
             clock=self.clock,
-            db_pool=db_pool,
             http_client=None,
             replication_layer=Mock(),
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
-            config=self.mock_config,
         )
         self.hs = hs
 

tests/rest/client/v2_alpha/__init__.py

@@ -17,11 +17,12 @@ from tests import unittest
 
 from mock import Mock
 
-from ....utils import MockHttpResource, MockKey
+from ....utils import MockHttpResource, setup_test_homeserver
 
-from synapse.server import HomeServer
 from synapse.types import UserID
 
+from twisted.internet import defer
+
 
 PATH_PREFIX = "/_matrix/client/v2_alpha"
 
@@ -31,19 +32,15 @@ class V2AlphaRestTestCase(unittest.TestCase):
     #   USER_ID = <some string>
     #   TO_REGISTER = [<list of REST servlets to register>]
 
+    @defer.inlineCallbacks
     def setUp(self):
         self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
 
-        mock_config = Mock()
+        hs = yield setup_test_homeserver(
-        mock_config.signing_key = [MockKey()]
-
-        hs = HomeServer("test",
-            db_pool=None,
             datastore=self.make_datastore_mock(),
             http_client=None,
             resource_for_client=self.mock_resource,
             resource_for_federation=self.mock_resource,
-            config=mock_config,
         )
 
         def _get_user_by_token(token=None):

tests/storage/test_base.py

@@ -22,6 +22,7 @@ from mock import Mock, call
 from collections import OrderedDict
 
 from synapse.server import HomeServer
+
 from synapse.storage._base import SQLBaseStore
 
 
@@ -37,8 +38,9 @@ class SQLBaseStoreTestCase(unittest.TestCase):
             return defer.succeed(func(self.mock_txn, *args, **kwargs))
         self.db_pool.runInteraction = runInteraction
 
-        hs = HomeServer("test",
+        config = Mock()
-                db_pool=self.db_pool)
+        config.event_cache_size = 1
+        hs = HomeServer("test", db_pool=self.db_pool, config=config)
 
         self.datastore = SQLBaseStore(hs)
 

tests/storage/test_directory.py

@@ -17,24 +17,17 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.storage.directory import DirectoryStore
 from synapse.types import RoomID, RoomAlias
 
-from tests.utils import SQLiteMemoryDbPool
+from tests.utils import setup_test_homeserver
 
 
 class DirectoryStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
+        hs = yield setup_test_homeserver()
-        yield db_pool.prepare()
-
-        hs = HomeServer(
-            "test",
-            db_pool=db_pool,
-        )
 
         self.store = DirectoryStore(hs)
 

tests/storage/test_presence.py

@@ -17,24 +17,17 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.storage.presence import PresenceStore
 from synapse.types import UserID
 
-from tests.utils import SQLiteMemoryDbPool, MockClock
+from tests.utils import setup_test_homeserver, MockClock
 
 
 class PresenceStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
+        hs = yield setup_test_homeserver(clock=MockClock())
-        yield db_pool.prepare()
-
-        hs = HomeServer("test",
-            clock=MockClock(),
-            db_pool=db_pool,
-        )
 
         self.store = PresenceStore(hs)
 

tests/storage/test_profile.py

@@ -17,23 +17,17 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.storage.profile import ProfileStore
 from synapse.types import UserID
 
-from tests.utils import SQLiteMemoryDbPool
+from tests.utils import setup_test_homeserver
 
 
 class ProfileStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
+        hs = yield setup_test_homeserver()
-        yield db_pool.prepare()
-
-        hs = HomeServer("test",
-            db_pool=db_pool,
-        )
 
         self.store = ProfileStore(hs)
 

tests/storage/test_redaction.py

@@ -17,11 +17,10 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.api.constants import EventTypes, Membership
 from synapse.types import UserID, RoomID
 
-from tests.utils import SQLiteMemoryDbPool, MockKey
+from tests.utils import setup_test_homeserver
 
 from mock import Mock
 
@@ -30,16 +29,7 @@ class RedactionTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
+        hs = yield setup_test_homeserver(
-        yield db_pool.prepare()
-
-        self.mock_config = Mock()
-        self.mock_config.signing_key = [MockKey()]
-
-        hs = HomeServer(
-            "test",
-            db_pool=db_pool,
-            config=self.mock_config,
             resource_for_federation=Mock(),
             http_client=None,
         )

tests/storage/test_registration.py

@@ -17,22 +17,16 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.storage.registration import RegistrationStore
 
-from tests.utils import SQLiteMemoryDbPool
+from tests.utils import setup_test_homeserver
 
 
 class RegistrationStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
+        hs = yield setup_test_homeserver()
-        yield db_pool.prepare()
-
-        hs = HomeServer("test",
-            db_pool=db_pool,
-        )
 
         self.store = RegistrationStore(hs)
 

tests/storage/test_room.py

@@ -17,23 +17,17 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.api.constants import EventTypes
 from synapse.types import UserID, RoomID, RoomAlias
 
-from tests.utils import SQLiteMemoryDbPool
+from tests.utils import setup_test_homeserver
 
 
 class RoomStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
+        hs = yield setup_test_homeserver()
-        yield db_pool.prepare()
-
-        hs = HomeServer("test",
-            db_pool=db_pool,
-        )
 
         # We can't test RoomStore on its own without the DirectoryStore, for
         # management of the 'room_aliases' table
@@ -86,12 +80,7 @@ class RoomEventsStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
+        hs = setup_test_homeserver()
-        yield db_pool.prepare()
-
-        hs = HomeServer("test",
-            db_pool=db_pool,
-        )
 
         # Room events need the full datastore, for persist_event() and
         # get_room_state()

tests/storage/test_roommember.py

@@ -17,11 +17,10 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.api.constants import EventTypes, Membership
 from synapse.types import UserID, RoomID
 
-from tests.utils import SQLiteMemoryDbPool, MockKey
+from tests.utils import setup_test_homeserver
 
 from mock import Mock
 
@@ -30,16 +29,7 @@ class RoomMemberStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
+        hs = yield setup_test_homeserver(
-        yield db_pool.prepare()
-
-        self.mock_config = Mock()
-        self.mock_config.signing_key = [MockKey()]
-
-        hs = HomeServer(
-            "test",
-            db_pool=db_pool,
-            config=self.mock_config,
             resource_for_federation=Mock(),
             http_client=None,
         )

tests/storage/test_stream.py

@@ -17,11 +17,10 @@
 from tests import unittest
 from twisted.internet import defer
 
-from synapse.server import HomeServer
 from synapse.api.constants import EventTypes, Membership
 from synapse.types import UserID, RoomID
 
-from tests.utils import SQLiteMemoryDbPool, MockKey
+from tests.utils import setup_test_homeserver
 
 from mock import Mock
 
@@ -30,16 +29,7 @@ class StreamStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
+        hs = yield setup_test_homeserver(
-        yield db_pool.prepare()
-
-        self.mock_config = Mock()
-        self.mock_config.signing_key = [MockKey()]
-
-        hs = HomeServer(
-            "test",
-            db_pool=db_pool,
-            config=self.mock_config,
             resource_for_federation=Mock(),
             http_client=None,
         )

tests/util/test_lrucache.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .. import unittest
+
+from synapse.util.lrucache import LruCache
+
+class LruCacheTestCase(unittest.TestCase):
+
+    def test_get_set(self):
+        cache = LruCache(1)
+        cache["key"] = "value"
+        self.assertEquals(cache.get("key"), "value")
+        self.assertEquals(cache["key"], "value")
+
+    def test_eviction(self):
+        cache = LruCache(2)
+        cache[1] = 1
+        cache[2] = 2
+
+        self.assertEquals(cache.get(1), 1)
+        self.assertEquals(cache.get(2), 2)
+
+        cache[3] = 3
+
+        self.assertEquals(cache.get(1), None)
+        self.assertEquals(cache.get(2), 2)
+        self.assertEquals(cache.get(3), 3)
+
+    def test_setdefault(self):
+        cache = LruCache(1)
+        self.assertEquals(cache.setdefault("key", 1), 1)
+        self.assertEquals(cache.get("key"), 1)
+        self.assertEquals(cache.setdefault("key", 2), 1)
+        self.assertEquals(cache.get("key"), 1)
+
+    def test_pop(self):
+        cache = LruCache(1)
+        cache["key"] = 1
+        self.assertEquals(cache.pop("key"), 1)
+        self.assertEquals(cache.pop("key"), None)
+
+

tests/utils.py

@@ -17,6 +17,7 @@ from synapse.http.server import HttpServer
 from synapse.api.errors import cs_error, CodeMessageException, StoreError
 from synapse.api.constants import EventTypes
 from synapse.storage import prepare_database
+from synapse.server import HomeServer
 
 from synapse.util.logcontext import LoggingContext
 
@@ -31,6 +32,29 @@ import urlparse
 from inspect import getcallargs
 
 
+@defer.inlineCallbacks
+def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
+    """Setup a homeserver suitable for running tests against. Keyword arguments
+    are passed to the Homeserver constructor. If no datastore is supplied a
+    datastore backed by an in-memory sqlite db will be given to the HS.
+    """
+    if config is None:
+        config = Mock()
+        config.signing_key = [MockKey()]
+        config.event_cache_size = 1
+
+    if datastore is None:
+        db_pool = SQLiteMemoryDbPool()
+        yield db_pool.prepare()
+        hs = HomeServer(name, db_pool=db_pool, config=config, **kargs)
+    else:
+        hs = HomeServer(
+            name, db_pool=None, datastore=datastore, config=config, **kargs
+        )
+
+    defer.returnValue(hs)
+
+
 def get_mock_call_args(pattern_func, mock_func):
     """ Return the arguments the mock function was called with interpreted
     by the pattern functions argument list.