From 763360594dfb90433f693056d3d64ac82409fc87 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 11 Feb 2016 14:10:00 +0000 Subject: [PATCH 01/31] Mark AS users with their AS's ID --- scripts/synapse_port_db | 9 +++- synapse/app/homeserver.py | 2 +- synapse/storage/appservice.py | 34 +++++++----- synapse/storage/engines/__init__.py | 5 +- synapse/storage/engines/postgres.py | 5 +- synapse/storage/engines/sqlite3.py | 5 +- synapse/storage/prepare_database.py | 15 +++--- synapse/storage/schema/delta/30/as_users.py | 59 +++++++++++++++++++++ tests/storage/test_base.py | 3 +- tests/utils.py | 20 ++++--- 10 files changed, 121 insertions(+), 36 deletions(-) create mode 100644 synapse/storage/schema/delta/30/as_users.py diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index fc92bbf2d..a2a0f364c 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -309,8 +309,8 @@ class Porter(object): **self.postgres_config["args"] ) - sqlite_engine = create_engine("sqlite3") - postgres_engine = create_engine("psycopg2") + sqlite_engine = create_engine(FakeConfig(sqlite_config)) + postgres_engine = create_engine(FakeConfig(postgres_config)) self.sqlite_store = Store(sqlite_db_pool, sqlite_engine) self.postgres_store = Store(postgres_db_pool, postgres_engine) @@ -792,3 +792,8 @@ if __name__ == "__main__": if end_error_exec_info: exc_type, exc_value, exc_traceback = end_error_exec_info traceback.print_exception(exc_type, exc_value, exc_traceback) + + +class FakeConfig: + def __init__(self, database_config): + self.database_config = database_config diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 2b4be7bdd..d2e758c40 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -382,7 +382,7 @@ def setup(config_options): tls_server_context_factory = context_factory.ServerContextFactory(config) - database_engine = create_engine(config.database_config["name"]) + database_engine = create_engine(config) config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection hs = SynapseHomeServer( diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 1100c6771..371600eeb 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -34,8 +34,8 @@ class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): super(ApplicationServiceStore, self).__init__(hs) self.hostname = hs.hostname - self.services_cache = [] - self._populate_appservice_cache( + self.services_cache = ApplicationServiceStore.load_appservices( + hs.hostname, hs.config.app_service_config_files ) @@ -144,21 +144,23 @@ class ApplicationServiceStore(SQLBaseStore): return rooms_for_user_matching_user_id - def _load_appservice(self, as_info): + @classmethod + def _load_appservice(cls, hostname, as_info, config_filename): required_string_fields = [ - # TODO: Add id here when it's stable to release - "url", "as_token", "hs_token", "sender_localpart" + "id", "url", "as_token", "hs_token", "sender_localpart" ] for field in required_string_fields: if not isinstance(as_info.get(field), basestring): - raise KeyError("Required string field: '%s'", field) + raise KeyError("Required string field: '%s' (%s)" % ( + field, config_filename, + )) localpart = as_info["sender_localpart"] if urllib.quote(localpart) != localpart: raise ValueError( "sender_localpart needs characters which are not URL encoded." ) - user = UserID(localpart, self.hostname) + user = UserID(localpart, hostname) user_id = user.to_string() # namespace checks @@ -188,25 +190,30 @@ class ApplicationServiceStore(SQLBaseStore): namespaces=as_info["namespaces"], hs_token=as_info["hs_token"], sender=user_id, - id=as_info["id"] if "id" in as_info else as_info["as_token"], + id=as_info["id"], ) - def _populate_appservice_cache(self, config_files): - """Populates a cache of Application Services from the config files.""" + @classmethod + def load_appservices(cls, hostname, config_files): + """Returns a list of Application Services from the config files.""" if not isinstance(config_files, list): logger.warning( "Expected %s to be a list of AS config files.", config_files ) - return + return [] # Dicts of value -> filename seen_as_tokens = {} seen_ids = {} + appservices = [] + for config_file in config_files: try: with open(config_file, 'r') as f: - appservice = self._load_appservice(yaml.load(f)) + appservice = ApplicationServiceStore._load_appservice( + hostname, yaml.load(f), config_file + ) if appservice.id in seen_ids: raise ConfigError( "Cannot reuse ID across application services: " @@ -226,11 +233,12 @@ class ApplicationServiceStore(SQLBaseStore): ) seen_as_tokens[appservice.token] = config_file logger.info("Loaded application service: %s", appservice) - self.services_cache.append(appservice) + appservices.append(appservice) except Exception as e: logger.error("Failed to load appservice from '%s'", config_file) logger.exception(e) raise + return appservices class ApplicationServiceTransactionStore(SQLBaseStore): diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 4290aea83..a48230b93 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -26,12 +26,13 @@ SUPPORTED_MODULE = { } -def create_engine(name): +def create_engine(config): + name = config.database_config["name"] engine_class = SUPPORTED_MODULE.get(name, None) if engine_class: module = importlib.import_module(name) - return engine_class(module) + return engine_class(module, config=config) raise RuntimeError( "Unsupported database engine '%s'" % (name,) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 17b7a9c07..a09685b4d 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -21,9 +21,10 @@ from ._base import IncorrectDatabaseSetup class PostgresEngine(object): single_threaded = False - def __init__(self, database_module): + def __init__(self, database_module, config): self.module = database_module self.module.extensions.register_type(self.module.extensions.UNICODE) + self.config = config def check_database(self, txn): txn.execute("SHOW SERVER_ENCODING") @@ -44,7 +45,7 @@ class PostgresEngine(object): ) def prepare_database(self, db_conn): - prepare_database(db_conn, self) + prepare_database(db_conn, self, config=self.config) def is_deadlock(self, error): if isinstance(error, self.module.DatabaseError): diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 91fac33b8..522b90594 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -23,8 +23,9 @@ import struct class Sqlite3Engine(object): single_threaded = True - def __init__(self, database_module): + def __init__(self, database_module, config): self.module = database_module + self.config = config def check_database(self, txn): pass @@ -38,7 +39,7 @@ class Sqlite3Engine(object): def prepare_database(self, db_conn): prepare_sqlite3_database(db_conn) - prepare_database(db_conn, self) + prepare_database(db_conn, self, config=self.config) def is_deadlock(self, error): return False diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 850736c85..3f29aad1e 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 29 +SCHEMA_VERSION = 30 dir_path = os.path.abspath(os.path.dirname(__file__)) @@ -50,7 +50,7 @@ class UpgradeDatabaseException(PrepareDatabaseException): pass -def prepare_database(db_conn, database_engine): +def prepare_database(db_conn, database_engine, config): """Prepares a database for usage. Will either create all necessary tables or upgrade from an older schema version. """ @@ -61,10 +61,10 @@ def prepare_database(db_conn, database_engine): if version_info: user_version, delta_files, upgraded = version_info _upgrade_existing_database( - cur, user_version, delta_files, upgraded, database_engine + cur, user_version, delta_files, upgraded, database_engine, config ) else: - _setup_new_database(cur, database_engine) + _setup_new_database(cur, database_engine, config) # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,)) @@ -75,7 +75,7 @@ def prepare_database(db_conn, database_engine): raise -def _setup_new_database(cur, database_engine): +def _setup_new_database(cur, database_engine, config): """Sets up the database by finding a base set of "full schemas" and then applying any necessary deltas. @@ -148,11 +148,12 @@ def _setup_new_database(cur, database_engine): applied_delta_files=[], upgraded=False, database_engine=database_engine, + config=config, ) def _upgrade_existing_database(cur, current_version, applied_delta_files, - upgraded, database_engine): + upgraded, database_engine, config): """Upgrades an existing database. Delta files can either be SQL stored in *.sql files, or python modules @@ -245,7 +246,7 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, module_name, absolute_path, python_file ) logger.debug("Running script %s", relative_path) - module.run_upgrade(cur, database_engine) + module.run_upgrade(cur, database_engine, config=config) elif ext == ".pyc": # Sometimes .pyc files turn up anyway even though we've # disabled their generation; e.g. from distribution package diff --git a/synapse/storage/schema/delta/30/as_users.py b/synapse/storage/schema/delta/30/as_users.py new file mode 100644 index 000000000..4cf4dd091 --- /dev/null +++ b/synapse/storage/schema/delta/30/as_users.py @@ -0,0 +1,59 @@ +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from synapse.storage.appservice import ApplicationServiceStore + + +logger = logging.getLogger(__name__) + + +def run_upgrade(cur, database_engine, config, *args, **kwargs): + # NULL indicates user was not registered by an appservice. + cur.execute("ALTER TABLE users ADD COLUMN appservice_id TEXT") + + cur.execute("SELECT name FROM users") + rows = cur.fetchall() + + config_files = [] + try: + config_files = config.app_service_config_files + except AttributeError: + logger.warning("Could not get app_service_config_files from config") + pass + + appservices = ApplicationServiceStore.load_appservices( + config.server_name, config_files + ) + + owned = {} + + for row in rows: + user_id = row[0] + for appservice in appservices: + if appservice.is_exclusive_user(user_id): + if user_id in owned.keys(): + logger.error( + "user_id %s was owned by more than one application" + " service (IDs %s and %s); assigning arbitrarily to %s" % + (user_id, owned[user_id], appservice.id, owned[user_id]) + ) + owned[user_id] = appservice.id + + for user_id, as_id in owned.items(): + cur.execute( + database_engine.convert_param_style( + "UPDATE users SET appservice_id = ? WHERE name = ?" + ), + (as_id, user_id) + ) diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 152d02766..0684fb6f7 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -48,11 +48,12 @@ class SQLBaseStoreTestCase(unittest.TestCase): config = Mock() config.event_cache_size = 1 + config.database_config = {"name": "sqlite3"} hs = HomeServer( "test", db_pool=self.db_pool, config=config, - database_engine=create_engine("sqlite3"), + database_engine=create_engine(config), ) self.datastore = SQLBaseStore(hs) diff --git a/tests/utils.py b/tests/utils.py index 3b1eb50d8..f3935648a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -51,6 +51,8 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config.server_name = "server.under.test" config.trusted_third_party_id_servers = [] + config.database_config = {"name": "sqlite3"} + if "clock" not in kargs: kargs["clock"] = MockClock() @@ -60,7 +62,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): hs = HomeServer( name, db_pool=db_pool, config=config, version_string="Synapse/tests", - database_engine=create_engine("sqlite3"), + database_engine=create_engine(config), get_db_conn=db_pool.get_db_conn, **kargs ) @@ -69,7 +71,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): hs = HomeServer( name, db_pool=None, datastore=datastore, config=config, version_string="Synapse/tests", - database_engine=create_engine("sqlite3"), + database_engine=create_engine(config), **kargs ) @@ -277,18 +279,24 @@ class SQLiteMemoryDbPool(ConnectionPool, object): cp_max=1, ) + self.config = Mock() + self.config.database_config = {"name": "sqlite3"} + def prepare(self): - engine = create_engine("sqlite3") + engine = self.create_engine() return self.runWithConnection( - lambda conn: prepare_database(conn, engine) + lambda conn: prepare_database(conn, engine, self.config) ) def get_db_conn(self): conn = self.connect() - engine = create_engine("sqlite3") - prepare_database(conn, engine) + engine = self.create_engine() + prepare_database(conn, engine, self.config) return conn + def create_engine(self): + return create_engine(self.config) + class MemoryDataStore(object): From 210b7d8e004e5d107a95572860dcc3d7c5b66fdd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 22 Feb 2016 22:55:21 +0100 Subject: [PATCH 02/31] handlers/_base: don't allow room create event to be changed Signed-off-by: Patrik Oldsberg --- synapse/handlers/_base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 5613bd205..5b27ec136 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -342,6 +342,12 @@ class BaseHandler(object): "You don't have permission to redact events" ) + if event.type == EventTypes.Create and context.current_state: + raise AuthError( + 403, + "Changing the room create event is forbidden", + ) + action_generator = ActionGenerator(self.hs) yield action_generator.handle_push_actions_for_event( event, context, self From 9c48f1ed224e02c30d73388c5e6a5839976bbc58 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 29 Feb 2016 23:11:06 +0100 Subject: [PATCH 03/31] handlers/register: make sure another user id is generated when a collision occurs Signed-off-by: Patrik Oldsberg --- synapse/handlers/register.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 6d155d57e..c5e5b2881 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -157,6 +157,7 @@ class RegistrationHandler(BaseHandler): ) except SynapseError: # if user id is taken, just generate another + user = None user_id = None token = None attempts += 1 From 910fc0f28f82ca7c719b5bbe98f4d3c2d0d76abf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Mar 2016 12:56:39 +0000 Subject: [PATCH 04/31] Add enviroment variable SYNAPSE_CACHE_FACTOR, default it to 0.1 --- synapse/handlers/presence.py | 4 ++++ synapse/handlers/receipts.py | 2 -- synapse/util/caches/descriptors.py | 6 ++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 08e38cdd2..a5c363e39 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -130,6 +130,10 @@ class PresenceHandler(BaseHandler): for state in active_presence } + metrics.register_callback( + "user_to_current_state_size", lambda: len(self.user_to_current_state) + ) + now = self.clock.time_msec() for state in active_presence: self.wheel_timer.insert( diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index de4c69471..935c33970 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -36,8 +36,6 @@ class ReceiptsHandler(BaseHandler): ) self.clock = self.hs.get_clock() - self._receipt_cache = None - @defer.inlineCallbacks def received_client_receipt(self, room_id, receipt_type, user_id, event_id): diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index 277854ccb..2b51ad6d0 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -28,6 +28,7 @@ from twisted.internet import defer from collections import OrderedDict +import os import functools import inspect import threading @@ -38,9 +39,14 @@ logger = logging.getLogger(__name__) _CacheSentinel = object() +CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.1)) + + class Cache(object): def __init__(self, name, max_entries=1000, keylen=1, lru=True, tree=False): + max_entries = int(max_entries * CACHE_SIZE_FACTOR) + if lru: cache_type = TreeCache if tree else dict self.cache = LruCache( From ce2cdced6136e81e182242c11c50e7b1c8a5a622 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Mar 2016 13:21:46 +0000 Subject: [PATCH 05/31] Move cache size fiddling to descriptors only. Fix tests --- synapse/util/caches/descriptors.py | 4 ++-- tests/storage/test_appservice.py | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index 2b51ad6d0..35544b19f 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -45,8 +45,6 @@ CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.1)) class Cache(object): def __init__(self, name, max_entries=1000, keylen=1, lru=True, tree=False): - max_entries = int(max_entries * CACHE_SIZE_FACTOR) - if lru: cache_type = TreeCache if tree else dict self.cache = LruCache( @@ -146,6 +144,8 @@ class CacheDescriptor(object): """ def __init__(self, orig, max_entries=1000, num_args=1, lru=True, tree=False, inlineCallbacks=False): + max_entries = int(max_entries * CACHE_SIZE_FACTOR) + self.orig = orig if inlineCallbacks: diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index ed8af10d8..573419812 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -35,7 +35,8 @@ class ApplicationServiceStoreTestCase(unittest.TestCase): def setUp(self): self.as_yaml_files = [] config = Mock( - app_service_config_files=self.as_yaml_files + app_service_config_files=self.as_yaml_files, + event_cache_size=1, ) hs = yield setup_test_homeserver(config=config) @@ -109,7 +110,8 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase): self.as_yaml_files = [] config = Mock( - app_service_config_files=self.as_yaml_files + app_service_config_files=self.as_yaml_files, + event_cache_size=1, ) hs = yield setup_test_homeserver(config=config) self.db_pool = hs.get_db_pool() @@ -438,7 +440,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase): f1 = self._write_config(suffix="1") f2 = self._write_config(suffix="2") - config = Mock(app_service_config_files=[f1, f2]) + config = Mock(app_service_config_files=[f1, f2], event_cache_size=1) hs = yield setup_test_homeserver(config=config, datastore=Mock()) ApplicationServiceStore(hs) @@ -448,7 +450,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase): f1 = self._write_config(id="id", suffix="1") f2 = self._write_config(id="id", suffix="2") - config = Mock(app_service_config_files=[f1, f2]) + config = Mock(app_service_config_files=[f1, f2], event_cache_size=1) hs = yield setup_test_homeserver(config=config, datastore=Mock()) with self.assertRaises(ConfigError) as cm: @@ -464,7 +466,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase): f1 = self._write_config(as_token="as_token", suffix="1") f2 = self._write_config(as_token="as_token", suffix="2") - config = Mock(app_service_config_files=[f1, f2]) + config = Mock(app_service_config_files=[f1, f2], event_cache_size=1) hs = yield setup_test_homeserver(config=config, datastore=Mock()) with self.assertRaises(ConfigError) as cm: From 374f9b2f073a8d7832916cf1b17d6aacba235066 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Mar 2016 13:30:15 +0000 Subject: [PATCH 06/31] Limit stream change cache size too --- synapse/util/caches/stream_change_cache.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py index 970488a19..a1aec7aa5 100644 --- a/synapse/util/caches/stream_change_cache.py +++ b/synapse/util/caches/stream_change_cache.py @@ -18,11 +18,15 @@ from synapse.util.caches import cache_counter, caches_by_name from blist import sorteddict import logging +import os logger = logging.getLogger(__name__) +CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.1)) + + class StreamChangeCache(object): """Keeps track of the stream positions of the latest change in a set of entities. @@ -33,7 +37,7 @@ class StreamChangeCache(object): old then the cache will simply return all given entities. """ def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache={}): - self._max_size = max_size + self._max_size = int(max_size * CACHE_SIZE_FACTOR) self._entity_to_key = {} self._cache = sorteddict() self._earliest_known_stream_pos = current_stream_pos From f9af8962f8ea6201ed3910eb248b8668f1262fef Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 1 Mar 2016 14:46:31 +0000 Subject: [PATCH 07/31] Allow alias creators to delete aliases --- synapse/handlers/directory.py | 27 ++++++++++++++----- synapse/rest/client/v1/directory.py | 3 --- synapse/storage/directory.py | 15 ++++++++++- .../storage/schema/delta/30/alias_creator.sql | 16 +++++++++++ 4 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 synapse/storage/schema/delta/30/alias_creator.sql diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index e0a778e7f..cce6f76f0 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -17,9 +17,9 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.api.errors import SynapseError, Codes, CodeMessageException +from synapse.api.errors import SynapseError, Codes, CodeMessageException, AuthError from synapse.api.constants import EventTypes -from synapse.types import RoomAlias +from synapse.types import RoomAlias, UserID import logging import string @@ -38,7 +38,7 @@ class DirectoryHandler(BaseHandler): ) @defer.inlineCallbacks - def _create_association(self, room_alias, room_id, servers=None): + def _create_association(self, room_alias, room_id, servers=None, creator=None): # general association creation for both human users and app services for wchar in string.whitespace: @@ -60,7 +60,8 @@ class DirectoryHandler(BaseHandler): yield self.store.create_room_alias_association( room_alias, room_id, - servers + servers, + creator=creator, ) @defer.inlineCallbacks @@ -77,7 +78,7 @@ class DirectoryHandler(BaseHandler): 400, "This alias is reserved by an application service.", errcode=Codes.EXCLUSIVE ) - yield self._create_association(room_alias, room_id, servers) + yield self._create_association(room_alias, room_id, servers, creator=user_id) @defer.inlineCallbacks def create_appservice_association(self, service, room_alias, room_id, @@ -95,7 +96,11 @@ class DirectoryHandler(BaseHandler): def delete_association(self, user_id, room_alias): # association deletion for human users - # TODO Check if server admin + can_delete = yield self._user_can_delete_alias(room_alias, user_id) + if not can_delete: + raise AuthError( + 403, "You don't have permission to delete the alias.", + ) can_delete = yield self.can_modify_alias( room_alias, @@ -257,3 +262,13 @@ class DirectoryHandler(BaseHandler): return # either no interested services, or no service with an exclusive lock defer.returnValue(True) + + @defer.inlineCallbacks + def _user_can_delete_alias(self, alias, user_id): + creator = yield self.store.get_room_alias_creator(alias.to_string()) + + if creator and creator == user_id: + defer.returnValue(True) + + is_admin = yield self.auth.is_server_admin(UserID.from_string(user_id)) + defer.returnValue(is_admin) diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 74ec1e50e..55c22000f 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -118,9 +118,6 @@ class ClientDirectoryServer(ClientV1RestServlet): requester = yield self.auth.get_user_by_req(request) user = requester.user - is_admin = yield self.auth.is_server_admin(user) - if not is_admin: - raise AuthError(403, "You need to be a server admin") room_alias = RoomAlias.from_string(room_alias) diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 1556619d5..012a0b414 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -70,13 +70,14 @@ class DirectoryStore(SQLBaseStore): ) @defer.inlineCallbacks - def create_room_alias_association(self, room_alias, room_id, servers): + def create_room_alias_association(self, room_alias, room_id, servers, creator=None): """ Creates an associatin between a room alias and room_id/servers Args: room_alias (RoomAlias) room_id (str) servers (list) + creator (str): Optional user_id of creator. Returns: Deferred @@ -87,6 +88,7 @@ class DirectoryStore(SQLBaseStore): { "room_alias": room_alias.to_string(), "room_id": room_id, + "creator": creator, }, desc="create_room_alias_association", ) @@ -107,6 +109,17 @@ class DirectoryStore(SQLBaseStore): ) self.get_aliases_for_room.invalidate((room_id,)) + def get_room_alias_creator(self, room_alias): + return self._simple_select_one_onecol( + table="room_aliases", + keyvalues={ + "room_alias": room_alias, + }, + retcol="creator", + desc="get_room_alias_creator", + allow_none=True + ) + @defer.inlineCallbacks def delete_room_alias(self, room_alias): room_id = yield self.runInteraction( diff --git a/synapse/storage/schema/delta/30/alias_creator.sql b/synapse/storage/schema/delta/30/alias_creator.sql new file mode 100644 index 000000000..c9d0dde63 --- /dev/null +++ b/synapse/storage/schema/delta/30/alias_creator.sql @@ -0,0 +1,16 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ALTER TABLE room_aliases ADD COLUMN creator TEXT; From 8a1d3b86af5a6b23bd45ed137a4eb2b53c274297 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 1 Mar 2016 17:27:22 +0000 Subject: [PATCH 08/31] Handle rejections of invites from local users locally Slightly hacky fix to SYN-642, which avoids the federation codepath when trying to reject invites from local users. --- synapse/handlers/_base.py | 16 +++++++++++++--- synapse/handlers/room.py | 23 ++++++++++++++++++----- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index bdade98bf..51bd1ee4d 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -199,8 +199,7 @@ class BaseHandler(object): # events in the room, because we don't know enough about the graph # fragment we received to treat it like a graph, so the above returned # no relevant events. It may have returned some events (if we have - # joined and left the room), but not useful ones, like the invite. So we - # forcibly set our context to the invite we received over federation. + # joined and left the room), but not useful ones, like the invite. if ( not self.is_host_in_room(context.current_state) and builder.type == EventTypes.Member @@ -208,7 +207,18 @@ class BaseHandler(object): prev_member_event = yield self.store.get_room_member( builder.sender, builder.room_id ) - if prev_member_event: + + # if we have the prev_member_event in context, we didn't receive + # the invite over federation. (More likely is that the inviting + # user, and all other local users, have left, making + # is_host_in_room return false). + # + context_events = (e.event_id for e in context.current_state.values()) + + if prev_member_event and not prev_member_event.event_id in context_events: + # The prev_member_event is missing from context, so it must + # have arrived over federation and is an outlier. We forcibly + # set our context to the invite we received over federation builder.prev_events = ( prev_member_event.event_id, prev_member_event.prev_events diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index d2de23a6c..714bff175 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -521,8 +521,24 @@ class RoomMemberHandler(BaseHandler): action = "remote_join" elif event.membership == Membership.LEAVE: is_host_in_room = self.is_host_in_room(context.current_state) + if not is_host_in_room: - action = "remote_reject" + # perhaps we've been invited + inviter = self.get_inviter(target_user.to_string(), context.current_state) + if not inviter: + raise SynapseError(404, "Not a known room") + + if inviter.domain == self.server_name: + # the inviter was on our server, but has now left. Carry on + # with the normal rejection codepath. + # + # This is a bit of a hack, because the room might still be + # active on other servers. + pass + else: + # send the rejection to the inviter's HS. + remote_room_hosts = [inviter.domain] + action = "remote_reject" federation_handler = self.hs.get_handlers().federation_handler @@ -541,11 +557,8 @@ class RoomMemberHandler(BaseHandler): event.content, ) elif action == "remote_reject": - inviter = self.get_inviter(target_user.to_string(), context.current_state) - if not inviter: - raise SynapseError(404, "No known servers") yield federation_handler.do_remotely_reject_invite( - [inviter.domain], + remote_room_hosts, room_id, event.user_id ) From 05ea111c47efaec7f24cbe4c9bfae972d4b1f87f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 1 Mar 2016 17:45:24 +0000 Subject: [PATCH 09/31] Fix pyflakes warning --- synapse/handlers/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 51bd1ee4d..c1f40a4ec 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -215,7 +215,7 @@ class BaseHandler(object): # context_events = (e.event_id for e in context.current_state.values()) - if prev_member_event and not prev_member_event.event_id in context_events: + if prev_member_event and prev_member_event.event_id not in context_events: # The prev_member_event is missing from context, so it must # have arrived over federation and is an outlier. We forcibly # set our context to the invite we received over federation From dda2058d90a3ad96f2d9f8fb36fc57f8eec55680 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 2 Mar 2016 14:03:10 +0000 Subject: [PATCH 10/31] Add SYNAPSE_CACHE_FACTOR to README --- README.rst | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 39a338c79..8a745259b 100644 --- a/README.rst +++ b/README.rst @@ -565,4 +565,21 @@ sphinxcontrib-napoleon:: Building internal API documentation:: python setup.py build_sphinx - \ No newline at end of file + + + +Halp!! Synapse eats all my RAM! +=============================== + +Synapse's architecture is quite RAM hungry currently - we deliberately +cache a lot of recent room data and metadata in RAM in order to speed up +common requests. We'll improve this in future, but for now the easiest +way to either reduce the RAM usage (at the risk of slowing things down) +is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment +variable. Roughly speaking, a SYNAPSE_CACHE_FACTOR of 1.0 will max out +at around 3-4GB of resident memory - this is what we currently run the +matrix.org on. The default setting is currently 0.1, which is probably +around a ~700MB footprint. You can dial it down further to 0.02 if +desired, which targets roughly ~512MB. Conversely you can dial it up if +you need performance for lots of users and have a box with a lot of RAM. + From 27185de75255191dcc9bf30b1bde9b6fc9a27100 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 2 Mar 2016 14:30:50 +0000 Subject: [PATCH 11/31] Set SYNAPSE_CACHE_FACTOR=1 in jenkins --- jenkins.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins.sh b/jenkins.sh index 359fd2277..9646ac0b0 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -3,6 +3,7 @@ : ${WORKSPACE:="$(pwd)"} export PYTHONDONTWRITEBYTECODE=yep +export SYNAPSE_CACHE_FACTOR=1 # Output test results as junit xml export TRIAL_FLAGS="--reporter=subunit" From 9ff940a0ef22304732f31fb3d2256afc2aca6bc4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 2 Mar 2016 15:40:30 +0000 Subject: [PATCH 12/31] Address review comments --- synapse/handlers/_base.py | 20 ++++++++++++++------ synapse/handlers/room.py | 5 +++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index c1f40a4ec..363b10e3b 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -208,14 +208,22 @@ class BaseHandler(object): builder.sender, builder.room_id ) - # if we have the prev_member_event in context, we didn't receive - # the invite over federation. (More likely is that the inviting - # user, and all other local users, have left, making - # is_host_in_room return false). + # The prev_member_event may already be in context.current_state, + # despite us not being present in the room; in particular, if + # inviting user, and all other local users, have already left. # - context_events = (e.event_id for e in context.current_state.values()) + # In that case, we have all the information we need, and we don't + # want to drop "context" - not least because we may need to handle + # the invite locally, which will require us to have the whole + # context (not just prev_member_event) to auth it. + # + context_event_ids = ( + e.event_id for e in context.current_state.values() + ) - if prev_member_event and prev_member_event.event_id not in context_events: + if (prev_member_event and + prev_member_event.event_id not in context_event_ids + ): # The prev_member_event is missing from context, so it must # have arrived over federation and is an outlier. We forcibly # set our context to the invite we received over federation diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 714bff175..de7239858 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -528,7 +528,7 @@ class RoomMemberHandler(BaseHandler): if not inviter: raise SynapseError(404, "Not a known room") - if inviter.domain == self.server_name: + if self.hs.is_mine(inviter): # the inviter was on our server, but has now left. Carry on # with the normal rejection codepath. # @@ -537,7 +537,8 @@ class RoomMemberHandler(BaseHandler): pass else: # send the rejection to the inviter's HS. - remote_room_hosts = [inviter.domain] + remote_room_hosts = remote_room_hosts or [] + remote_room_hosts.append(inviter.domain) action = "remote_reject" federation_handler = self.hs.get_handlers().federation_handler From 863d3f26b326b1129cbf7edfb8c5344f3e25a849 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 2 Mar 2016 15:52:50 +0000 Subject: [PATCH 13/31] fix pyflakes quibble --- synapse/handlers/_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 363b10e3b..a6f890e0b 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -221,7 +221,8 @@ class BaseHandler(object): e.event_id for e in context.current_state.values() ) - if (prev_member_event and + if ( + prev_member_event and prev_member_event.event_id not in context_event_ids ): # The prev_member_event is missing from context, so it must From 47c361d2f88879d5e4b042a49282c93d7e38f9fb Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 2 Mar 2016 15:57:54 +0000 Subject: [PATCH 14/31] add 800x600 thumbnails to make vector look prettier (and anyone else who likes big thumbnails) --- synapse/config/repository.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 2fcf87244..2e96c0901 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -97,4 +97,7 @@ class ContentRepositoryConfig(Config): - width: 640 height: 480 method: scale + - width: 800 + height: 600 + method: scale """ % locals() From fc1f932cc04ed6675fb839b58d81d5ac6c23b4c5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 2 Mar 2016 16:44:14 +0000 Subject: [PATCH 15/31] Move arg default to the start of the function Also don't overwrite the list that gets passed in. --- synapse/handlers/room.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index de7239858..9ef3219e3 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -493,6 +493,8 @@ class RoomMemberHandler(BaseHandler): Raises: SynapseError if there was a problem changing the membership. """ + remote_room_hosts = remote_room_hosts or [] + target_user = UserID.from_string(event.state_key) room_id = event.room_id @@ -537,8 +539,7 @@ class RoomMemberHandler(BaseHandler): pass else: # send the rejection to the inviter's HS. - remote_room_hosts = remote_room_hosts or [] - remote_room_hosts.append(inviter.domain) + remote_room_hosts = remote_room_hosts + [inviter.doman] action = "remote_reject" federation_handler = self.hs.get_handlers().federation_handler From ff8b87118dcfb153d972e29c2b77b195244d5ddc Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 2 Mar 2016 18:06:45 +0000 Subject: [PATCH 16/31] Stop using checkpw as it seems to have vanished from bcrypt. Use `bcrypt.hashpw(password, hashed) == hashed` as per the bcrypt README. --- synapse/handlers/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 62e82a257..7a4afe446 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -477,4 +477,4 @@ class AuthHandler(BaseHandler): Returns: Whether self.hash(password) == stored_hash (bool). """ - return bcrypt.checkpw(password, stored_hash) + return bcrypt.hashpw(password, stored_hash) == stored_hash From 74cd80e53026d9f603e9e4f9a41cef54de239e16 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 3 Mar 2016 10:28:58 +0000 Subject: [PATCH 17/31] Fix typo --- synapse/handlers/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 9ef3219e3..ad7c83f47 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -539,7 +539,7 @@ class RoomMemberHandler(BaseHandler): pass else: # send the rejection to the inviter's HS. - remote_room_hosts = remote_room_hosts + [inviter.doman] + remote_room_hosts = remote_room_hosts + [inviter.domain] action = "remote_reject" federation_handler = self.hs.get_handlers().federation_handler From b139e510412430cb7decd58a098871f7ded9ef0c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 3 Mar 2016 11:38:36 +0000 Subject: [PATCH 18/31] jenkins.sh: set -x Also move the options from the shebang line to the body of the script, so that they take effect even if somebody explicitly runs "bash jenkins.sh" --- jenkins.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins.sh b/jenkins.sh index 9646ac0b0..b826d510c 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -1,4 +1,6 @@ -#!/bin/bash -eu +#!/bin/bash + +set -eux : ${WORKSPACE:="$(pwd)"} From 690596b770c75838d0a64c7b522369d61a9a66c7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 3 Mar 2016 14:47:40 +0000 Subject: [PATCH 19/31] Add jenkins-sqlite.sh --- jenkins-sqlite.sh | 61 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100755 jenkins-sqlite.sh diff --git a/jenkins-sqlite.sh b/jenkins-sqlite.sh new file mode 100755 index 000000000..1e5a1dbae --- /dev/null +++ b/jenkins-sqlite.sh @@ -0,0 +1,61 @@ +#!/bin/bash -eu + +: ${WORKSPACE:="$(pwd)"} + +export PYTHONDONTWRITEBYTECODE=yep +export SYNAPSE_CACHE_FACTOR=1 + +# Output test results as junit xml +export TRIAL_FLAGS="--reporter=subunit" +export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml" +# Write coverage reports to a separate file for each process +export COVERAGE_OPTS="-p" +export DUMP_COVERAGE_COMMAND="coverage help" + +# Output flake8 violations to violations.flake8.log +# Don't exit with non-0 status code on Jenkins, +# so that the build steps continue and a later step can decided whether to +# UNSTABLE or FAILURE this build. +export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?" + +rm .coverage* || echo "No coverage files to remove" + +tox --notest + +: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"} + +TOX_BIN=$WORKSPACE/.tox/py27/bin + +if [[ ! -e .sytest-base ]]; then + git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror +else + (cd .sytest-base; git fetch -p) +fi + +rm -rf sytest +git clone .sytest-base sytest --shared +cd sytest + +git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop) + +: ${PERL5LIB:=$WORKSPACE/perl5/lib/perl5} +: ${PERL_MB_OPT:=--install_base=$WORKSPACE/perl5} +: ${PERL_MM_OPT:=INSTALL_BASE=$WORKSPACE/perl5} +export PERL5LIB PERL_MB_OPT PERL_MM_OPT + +./install-deps.pl + +: ${PORT_BASE:=8000} + +echo >&2 "Running sytest with SQLite3"; +./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \ + --python $TOX_BIN/python --all --port-base $PORT_BASE > results-sqlite3.tap + +cd .. +cp sytest/.coverage.* . + +# Combine the coverage reports +echo "Combining:" .coverage.* +$TOX_BIN/python -m coverage combine +# Output coverage to coverage.xml +$TOX_BIN/coverage xml -o coverage.xml From 91f4ac602bbf63fcdd88eb4f5668f998dfd6c599 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 3 Mar 2016 14:51:09 +0000 Subject: [PATCH 20/31] Exclude all jenkins build scripts --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 5668665db..211bde2fc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -21,5 +21,6 @@ recursive-include synapse/static *.html recursive-include synapse/static *.js exclude jenkins.sh +exclude jenkins*.sh prune demo/etc From 6789b63131eb36532a0be0aa7a376c1683037afa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 3 Mar 2016 15:02:07 +0000 Subject: [PATCH 21/31] Use different PORT_BASE --- jenkins-sqlite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-sqlite.sh b/jenkins-sqlite.sh index 1e5a1dbae..20209cf7b 100755 --- a/jenkins-sqlite.sh +++ b/jenkins-sqlite.sh @@ -45,7 +45,7 @@ export PERL5LIB PERL_MB_OPT PERL_MM_OPT ./install-deps.pl -: ${PORT_BASE:=8000} +: ${PORT_BASE:=9000} echo >&2 "Running sytest with SQLite3"; ./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \ From 246b8c6e4a7e2f568e81f994ed2d0662db952732 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 3 Mar 2016 15:16:19 +0000 Subject: [PATCH 22/31] Change port-base in jenkins-sqlite.sh --- jenkins-sqlite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-sqlite.sh b/jenkins-sqlite.sh index 20209cf7b..2d98a0af9 100755 --- a/jenkins-sqlite.sh +++ b/jenkins-sqlite.sh @@ -45,7 +45,7 @@ export PERL5LIB PERL_MB_OPT PERL_MM_OPT ./install-deps.pl -: ${PORT_BASE:=9000} +: ${PORT_BASE:=8500} echo >&2 "Running sytest with SQLite3"; ./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \ From fc9c7b6cbc27b17aa08cc77832249a00cdbe0338 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 3 Mar 2016 15:26:07 +0000 Subject: [PATCH 23/31] Add jenkins-postgres.sh --- jenkins-postgres.sh | 87 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100755 jenkins-postgres.sh diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh new file mode 100755 index 000000000..7dca840e3 --- /dev/null +++ b/jenkins-postgres.sh @@ -0,0 +1,87 @@ +#!/bin/bash -eu + +: ${WORKSPACE:="$(pwd)"} + +export PYTHONDONTWRITEBYTECODE=yep +export SYNAPSE_CACHE_FACTOR=1 + +# Output test results as junit xml +export TRIAL_FLAGS="--reporter=subunit" +export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml" +# Write coverage reports to a separate file for each process +export COVERAGE_OPTS="-p" +export DUMP_COVERAGE_COMMAND="coverage help" + +# Output flake8 violations to violations.flake8.log +# Don't exit with non-0 status code on Jenkins, +# so that the build steps continue and a later step can decided whether to +# UNSTABLE or FAILURE this build. +export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?" + +rm .coverage* || echo "No coverage files to remove" + +tox --notest + +: ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"} + +TOX_BIN=$WORKSPACE/.tox/py27/bin + +if [[ ! -e .sytest-base ]]; then + git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror +else + (cd .sytest-base; git fetch -p) +fi + +rm -rf sytest +git clone .sytest-base sytest --shared +cd sytest + +git checkout "${GIT_BRANCH}" || (echo >&2 "No ref ${GIT_BRANCH} found, falling back to develop" ; git checkout develop) + +: ${PERL5LIB:=$WORKSPACE/perl5/lib/perl5} +: ${PERL_MB_OPT:=--install_base=$WORKSPACE/perl5} +: ${PERL_MM_OPT:=INSTALL_BASE=$WORKSPACE/perl5} +export PERL5LIB PERL_MB_OPT PERL_MM_OPT + +./install-deps.pl + +: ${PORT_BASE:=8000} + + +if [[ -z "$POSTGRES_DB_1" ]]; then + echo >&2 "Variable POSTGRES_DB_1 not set" + exit 1 +fi + +if [[ -z "$POSTGRES_DB_2" ]]; then + echo >&2 "Variable POSTGRES_DB_2 not set" + exit 1 +fi + +cat > localhost-$(($PORT_BASE + 1))/database.yaml << EOF +name: psycopg2 +args: + database: $POSTGRES_DB_1 +EOF + +cat > localhost-$(($PORT_BASE + 2))/database.yaml << EOF +name: psycopg2 +args: + database: $POSTGRES_DB_2 +EOF + + +# Run if both postgresql databases exist +echo >&2 "Running sytest with PostgreSQL"; +$TOX_BIN/pip install psycopg2 +./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \ + --python $TOX_BIN/python --all --port-base $PORT_BASE > results-postgresql.tap + +cd .. +cp sytest/.coverage.* . + +# Combine the coverage reports +echo "Combining:" .coverage.* +$TOX_BIN/python -m coverage combine +# Output coverage to coverage.xml +$TOX_BIN/coverage xml -o coverage.xml From 7678ec3f9b9a529c7d8c8aa8a38572c5cf2283d6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 3 Mar 2016 15:42:07 +0000 Subject: [PATCH 24/31] Mkdir --- jenkins-postgres.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh index 7dca840e3..1708cbfaa 100755 --- a/jenkins-postgres.sh +++ b/jenkins-postgres.sh @@ -58,6 +58,9 @@ if [[ -z "$POSTGRES_DB_2" ]]; then exit 1 fi +mkdir -p "localhost-$(($PORT_BASE + 1))" +mkdir -p "localhost-$(($PORT_BASE + 2))" + cat > localhost-$(($PORT_BASE + 1))/database.yaml << EOF name: psycopg2 args: From c037170faa6fcda689fdd8c3777cd9ef64c24016 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 3 Mar 2016 16:12:45 +0000 Subject: [PATCH 25/31] Split up jenkins tests --- jenkins-flake8.sh | 20 ++++++++++++++++++++ jenkins-unittests.sh | 23 +++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100755 jenkins-flake8.sh create mode 100755 jenkins-unittests.sh diff --git a/jenkins-flake8.sh b/jenkins-flake8.sh new file mode 100755 index 000000000..cbcb0ae4c --- /dev/null +++ b/jenkins-flake8.sh @@ -0,0 +1,20 @@ +#!/bin/bash -eu + +: ${WORKSPACE:="$(pwd)"} + +export PYTHONDONTWRITEBYTECODE=yep +export SYNAPSE_CACHE_FACTOR=1 + +# Output test results as junit xml +export TRIAL_FLAGS="--reporter=subunit" +export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml" +# Write coverage reports to a separate file for each process +export COVERAGE_OPTS="-p" +export DUMP_COVERAGE_COMMAND="coverage help" + +# Output flake8 violations to violations.flake8.log +export PEP8SUFFIX="--output-file=violations.flake8.log" + +rm .coverage* || echo "No coverage files to remove" + +tox -e packaging -e pep8 diff --git a/jenkins-unittests.sh b/jenkins-unittests.sh new file mode 100755 index 000000000..2fa2f1b1d --- /dev/null +++ b/jenkins-unittests.sh @@ -0,0 +1,23 @@ +#!/bin/bash -eu + +: ${WORKSPACE:="$(pwd)"} + +export PYTHONDONTWRITEBYTECODE=yep +export SYNAPSE_CACHE_FACTOR=1 + +# Output test results as junit xml +export TRIAL_FLAGS="--reporter=subunit" +export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml" +# Write coverage reports to a separate file for each process +export COVERAGE_OPTS="-p" +export DUMP_COVERAGE_COMMAND="coverage help" + +# Output flake8 violations to violations.flake8.log +# Don't exit with non-0 status code on Jenkins, +# so that the build steps continue and a later step can decided whether to +# UNSTABLE or FAILURE this build. +export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?" + +rm .coverage* || echo "No coverage files to remove" + +tox -e py27 From b4022cc487921ec46942a6a72fb174bb7aa1e459 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 3 Mar 2016 16:43:42 +0000 Subject: [PATCH 26/31] Pass whole requester to ratelimiting This will enable more detailed decisions --- synapse/handlers/_base.py | 15 ++++-- synapse/handlers/directory.py | 20 +++++--- synapse/handlers/federation.py | 4 +- synapse/handlers/message.py | 8 +-- synapse/handlers/profile.py | 17 ++++--- synapse/handlers/room.py | 76 ++++++++++++++++------------ synapse/rest/client/v1/directory.py | 6 ++- synapse/rest/client/v1/profile.py | 4 +- synapse/rest/client/v1/room.py | 8 +-- tests/handlers/test_profile.py | 16 ++++-- tests/replication/test_resource.py | 17 ++++--- tests/rest/client/v1/test_profile.py | 4 +- tests/utils.py | 5 ++ 13 files changed, 124 insertions(+), 76 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index bdade98bf..2333fc0c0 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -160,10 +160,10 @@ class BaseHandler(object): ) defer.returnValue(res.get(user_id, [])) - def ratelimit(self, user_id): + def ratelimit(self, requester): time_now = self.clock.time() allowed, time_allowed = self.ratelimiter.send_message( - user_id, time_now, + requester.user.to_string(), time_now, msg_rate_hz=self.hs.config.rc_messages_per_second, burst_count=self.hs.config.rc_message_burst_count, ) @@ -263,11 +263,18 @@ class BaseHandler(object): return False @defer.inlineCallbacks - def handle_new_client_event(self, event, context, ratelimit=True, extra_users=[]): + def handle_new_client_event( + self, + requester, + event, + context, + ratelimit=True, + extra_users=[] + ): # We now need to go and hit out to wherever we need to hit out to. if ratelimit: - self.ratelimit(event.sender) + self.ratelimit(requester) self.auth.check(event, auth_events=context.current_state) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index e0a778e7f..88166f018 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -212,17 +212,21 @@ class DirectoryHandler(BaseHandler): ) @defer.inlineCallbacks - def send_room_alias_update_event(self, user_id, room_id): + def send_room_alias_update_event(self, requester, user_id, room_id): aliases = yield self.store.get_aliases_for_room(room_id) msg_handler = self.hs.get_handlers().message_handler - yield msg_handler.create_and_send_nonmember_event({ - "type": EventTypes.Aliases, - "state_key": self.hs.hostname, - "room_id": room_id, - "sender": user_id, - "content": {"aliases": aliases}, - }, ratelimit=False) + yield msg_handler.create_and_send_nonmember_event( + requester, + { + "type": EventTypes.Aliases, + "state_key": self.hs.hostname, + "room_id": room_id, + "sender": user_id, + "content": {"aliases": aliases}, + }, + ratelimit=False + ) @defer.inlineCallbacks def get_association_from_room_alias(self, room_alias): diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 3655b9e5e..6e50b0963 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1657,7 +1657,7 @@ class FederationHandler(BaseHandler): self.auth.check(event, context.current_state) yield self._check_signature(event, auth_events=context.current_state) member_handler = self.hs.get_handlers().room_member_handler - yield member_handler.send_membership_event(event, context, from_client=False) + yield member_handler.send_membership_event(None, event, context) else: destinations = set(x.split(":", 1)[-1] for x in (sender_user_id, room_id)) yield self.replication_layer.forward_third_party_invite( @@ -1686,7 +1686,7 @@ class FederationHandler(BaseHandler): # TODO: Make sure the signatures actually are correct. event.signatures.update(returned_invite.signatures) member_handler = self.hs.get_handlers().room_member_handler - yield member_handler.send_membership_event(event, context, from_client=False) + yield member_handler.send_membership_event(None, event, context) @defer.inlineCallbacks def add_display_name_to_third_party_invite(self, event_dict, event, context): diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index afa7c9c36..cace1cb82 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -215,7 +215,7 @@ class MessageHandler(BaseHandler): defer.returnValue((event, context)) @defer.inlineCallbacks - def send_nonmember_event(self, event, context, ratelimit=True): + def send_nonmember_event(self, requester, event, context, ratelimit=True): """ Persists and notifies local clients and federation of an event. @@ -241,6 +241,7 @@ class MessageHandler(BaseHandler): defer.returnValue(prev_state) yield self.handle_new_client_event( + requester=requester, event=event, context=context, ratelimit=ratelimit, @@ -268,9 +269,9 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def create_and_send_nonmember_event( self, + requester, event_dict, ratelimit=True, - token_id=None, txn_id=None ): """ @@ -280,10 +281,11 @@ class MessageHandler(BaseHandler): """ event, context = yield self.create_event( event_dict, - token_id=token_id, + token_id=requester.access_token_id, txn_id=txn_id ) yield self.send_nonmember_event( + requester, event, context, ratelimit=ratelimit, diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index c9ad5944e..b45eafbb4 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -89,13 +89,13 @@ class ProfileHandler(BaseHandler): defer.returnValue(result["displayname"]) @defer.inlineCallbacks - def set_displayname(self, target_user, auth_user, new_displayname): + def set_displayname(self, target_user, requester, new_displayname): """target_user is the user whose displayname is to be changed; auth_user is the user attempting to make this change.""" if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this Home Server") - if target_user != auth_user: + if target_user != requester.user: raise AuthError(400, "Cannot set another user's displayname") if new_displayname == '': @@ -109,7 +109,7 @@ class ProfileHandler(BaseHandler): "displayname": new_displayname, }) - yield self._update_join_states(target_user) + yield self._update_join_states(requester) @defer.inlineCallbacks def get_avatar_url(self, target_user): @@ -139,13 +139,13 @@ class ProfileHandler(BaseHandler): defer.returnValue(result["avatar_url"]) @defer.inlineCallbacks - def set_avatar_url(self, target_user, auth_user, new_avatar_url): + def set_avatar_url(self, target_user, requester, new_avatar_url): """target_user is the user whose avatar_url is to be changed; auth_user is the user attempting to make this change.""" if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this Home Server") - if target_user != auth_user: + if target_user != requester.user: raise AuthError(400, "Cannot set another user's avatar_url") yield self.store.set_profile_avatar_url( @@ -156,7 +156,7 @@ class ProfileHandler(BaseHandler): "avatar_url": new_avatar_url, }) - yield self._update_join_states(target_user) + yield self._update_join_states(requester) @defer.inlineCallbacks def collect_presencelike_data(self, user, state): @@ -199,11 +199,12 @@ class ProfileHandler(BaseHandler): defer.returnValue(response) @defer.inlineCallbacks - def _update_join_states(self, user): + def _update_join_states(self, requester): + user = requester.user if not self.hs.is_mine(user): return - self.ratelimit(user.to_string()) + self.ratelimit(requester) joins = yield self.store.get_rooms_for_user( user.to_string(), diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index d2de23a6c..91fe306cf 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -18,7 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler -from synapse.types import UserID, RoomAlias, RoomID, RoomStreamToken +from synapse.types import UserID, RoomAlias, RoomID, RoomStreamToken, Requester from synapse.api.constants import ( EventTypes, Membership, JoinRules, RoomCreationPreset, ) @@ -90,7 +90,7 @@ class RoomCreationHandler(BaseHandler): """ user_id = requester.user.to_string() - self.ratelimit(user_id) + self.ratelimit(requester) if "room_alias_name" in config: for wchar in string.whitespace: @@ -185,23 +185,29 @@ class RoomCreationHandler(BaseHandler): if "name" in config: name = config["name"] - yield msg_handler.create_and_send_nonmember_event({ - "type": EventTypes.Name, - "room_id": room_id, - "sender": user_id, - "state_key": "", - "content": {"name": name}, - }, ratelimit=False) + yield msg_handler.create_and_send_nonmember_event( + requester, + { + "type": EventTypes.Name, + "room_id": room_id, + "sender": user_id, + "state_key": "", + "content": {"name": name}, + }, + ratelimit=False) if "topic" in config: topic = config["topic"] - yield msg_handler.create_and_send_nonmember_event({ - "type": EventTypes.Topic, - "room_id": room_id, - "sender": user_id, - "state_key": "", - "content": {"topic": topic}, - }, ratelimit=False) + yield msg_handler.create_and_send_nonmember_event( + requester, + { + "type": EventTypes.Topic, + "room_id": room_id, + "sender": user_id, + "state_key": "", + "content": {"topic": topic}, + }, + ratelimit=False) for invitee in invite_list: room_member_handler.update_membership( @@ -231,7 +237,7 @@ class RoomCreationHandler(BaseHandler): if room_alias: result["room_alias"] = room_alias.to_string() yield directory_handler.send_room_alias_update_event( - user_id, room_id + requester, user_id, room_id ) defer.returnValue(result) @@ -263,7 +269,11 @@ class RoomCreationHandler(BaseHandler): @defer.inlineCallbacks def send(etype, content, **kwargs): event = create(etype, content, **kwargs) - yield msg_handler.create_and_send_nonmember_event(event, ratelimit=False) + yield msg_handler.create_and_send_nonmember_event( + creator, + event, + ratelimit=False + ) config = RoomCreationHandler.PRESETS_DICT[preset_config] @@ -454,12 +464,11 @@ class RoomMemberHandler(BaseHandler): member_handler = self.hs.get_handlers().room_member_handler yield member_handler.send_membership_event( + requester, event, context, - is_guest=requester.is_guest, ratelimit=ratelimit, remote_room_hosts=remote_room_hosts, - from_client=True, ) if action == "forget": @@ -468,17 +477,19 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def send_membership_event( self, + requester, event, context, - is_guest=False, remote_room_hosts=None, ratelimit=True, - from_client=True, ): """ Change the membership status of a user in a room. Args: + requester (Requester): The local user who requested the membership + event. If None, certain checks, like whether this homeserver can + act as the sender, will be skipped. event (SynapseEvent): The membership event. context: The context of the event. is_guest (bool): Whether the sender is a guest. @@ -486,19 +497,21 @@ class RoomMemberHandler(BaseHandler): the room, and could be danced with in order to join this homeserver for the first time. ratelimit (bool): Whether to rate limit this request. - from_client (bool): Whether this request is the result of a local - client request (rather than over federation). If so, we will - perform extra checks, like that this homeserver can act as this - client. Raises: SynapseError if there was a problem changing the membership. """ target_user = UserID.from_string(event.state_key) room_id = event.room_id - if from_client: + if requester is not None: sender = UserID.from_string(event.sender) + assert sender == requester.user, ( + "Sender (%s) must be same as requester (%s)" % + (sender, requester.user) + ) assert self.hs.is_mine(sender), "Sender must be our own: %s" % (sender,) + else: + requester = Requester(target_user, None, False) message_handler = self.hs.get_handlers().message_handler prev_event = message_handler.deduplicate_state_event(event, context) @@ -508,7 +521,7 @@ class RoomMemberHandler(BaseHandler): action = "send" if event.membership == Membership.JOIN: - if is_guest and not self._can_guest_join(context.current_state): + if requester.is_guest and not self._can_guest_join(context.current_state): # This should be an auth check, but guests are a local concept, # so don't really fit into the general auth process. raise AuthError(403, "Guest access not allowed") @@ -551,6 +564,7 @@ class RoomMemberHandler(BaseHandler): ) else: yield self.handle_new_client_event( + requester, event, context, extra_users=[target_user], @@ -669,12 +683,12 @@ class RoomMemberHandler(BaseHandler): ) else: yield self._make_and_store_3pid_invite( + requester, id_server, medium, address, room_id, inviter, - requester.access_token_id, txn_id=txn_id ) @@ -732,12 +746,12 @@ class RoomMemberHandler(BaseHandler): @defer.inlineCallbacks def _make_and_store_3pid_invite( self, + requester, id_server, medium, address, room_id, user, - token_id, txn_id ): room_state = yield self.hs.get_state_handler().get_current_state(room_id) @@ -787,6 +801,7 @@ class RoomMemberHandler(BaseHandler): msg_handler = self.hs.get_handlers().message_handler yield msg_handler.create_and_send_nonmember_event( + requester, { "type": EventTypes.ThirdPartyInvite, "content": { @@ -801,7 +816,6 @@ class RoomMemberHandler(BaseHandler): "sender": user.to_string(), "state_key": token, }, - token_id=token_id, txn_id=txn_id, ) diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 74ec1e50e..8c1a2614a 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -75,7 +75,11 @@ class ClientDirectoryServer(ClientV1RestServlet): yield dir_handler.create_association( user_id, room_alias, room_id, servers ) - yield dir_handler.send_room_alias_update_event(user_id, room_id) + yield dir_handler.send_room_alias_update_event( + requester, + user_id, + room_id + ) except SynapseError as e: raise e except: diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py index 3c5a21292..953764bd8 100644 --- a/synapse/rest/client/v1/profile.py +++ b/synapse/rest/client/v1/profile.py @@ -51,7 +51,7 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet): defer.returnValue((400, "Unable to parse name")) yield self.handlers.profile_handler.set_displayname( - user, requester.user, new_name) + user, requester, new_name) defer.returnValue((200, {})) @@ -88,7 +88,7 @@ class ProfileAvatarURLRestServlet(ClientV1RestServlet): defer.returnValue((400, "Unable to parse name")) yield self.handlers.profile_handler.set_avatar_url( - user, requester.user, new_name) + user, requester, new_name) defer.returnValue((200, {})) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index f5ed4f730..cbf3673ef 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -158,12 +158,12 @@ class RoomStateEventRestServlet(ClientV1RestServlet): if event_type == EventTypes.Member: yield self.handlers.room_member_handler.send_membership_event( + requester, event, context, - is_guest=requester.is_guest, ) else: - yield msg_handler.send_nonmember_event(event, context) + yield msg_handler.send_nonmember_event(requester, event, context) defer.returnValue((200, {"event_id": event.event_id})) @@ -183,13 +183,13 @@ class RoomSendEventRestServlet(ClientV1RestServlet): msg_handler = self.handlers.message_handler event = yield msg_handler.create_and_send_nonmember_event( + requester, { "type": event_type, "content": content, "room_id": room_id, "sender": requester.user.to_string(), }, - token_id=requester.access_token_id, txn_id=txn_id, ) @@ -504,6 +504,7 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): msg_handler = self.handlers.message_handler event = yield msg_handler.create_and_send_nonmember_event( + requester, { "type": EventTypes.Redaction, "content": content, @@ -511,7 +512,6 @@ class RoomRedactEventRestServlet(ClientV1RestServlet): "sender": requester.user.to_string(), "redacts": event_id, }, - token_id=requester.access_token_id, txn_id=txn_id, ) diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index a87703bbf..4f2c14e4f 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -23,7 +23,7 @@ from synapse.api.errors import AuthError from synapse.handlers.profile import ProfileHandler from synapse.types import UserID -from tests.utils import setup_test_homeserver +from tests.utils import setup_test_homeserver, requester_for_user class ProfileHandlers(object): @@ -84,7 +84,11 @@ class ProfileTestCase(unittest.TestCase): @defer.inlineCallbacks def test_set_my_name(self): - yield self.handler.set_displayname(self.frank, self.frank, "Frank Jr.") + yield self.handler.set_displayname( + self.frank, + requester_for_user(self.frank), + "Frank Jr." + ) self.assertEquals( (yield self.store.get_profile_displayname(self.frank.localpart)), @@ -93,7 +97,11 @@ class ProfileTestCase(unittest.TestCase): @defer.inlineCallbacks def test_set_my_name_noauth(self): - d = self.handler.set_displayname(self.frank, self.bob, "Frank Jr.") + d = self.handler.set_displayname( + self.frank, + requester_for_user(self.bob), + "Frank Jr." + ) yield self.assertFailure(d, AuthError) @@ -136,7 +144,7 @@ class ProfileTestCase(unittest.TestCase): @defer.inlineCallbacks def test_set_my_avatar(self): yield self.handler.set_avatar_url( - self.frank, self.frank, "http://my.server/pic.gif" + self.frank, requester_for_user(self.frank), "http://my.server/pic.gif" ) self.assertEquals( diff --git a/tests/replication/test_resource.py b/tests/replication/test_resource.py index 38daaf87e..daabc563b 100644 --- a/tests/replication/test_resource.py +++ b/tests/replication/test_resource.py @@ -18,7 +18,7 @@ from synapse.types import Requester, UserID from twisted.internet import defer from tests import unittest -from tests.utils import setup_test_homeserver +from tests.utils import setup_test_homeserver, requester_for_user from mock import Mock, NonCallableMock import json import contextlib @@ -133,12 +133,15 @@ class ReplicationResourceCase(unittest.TestCase): @defer.inlineCallbacks def send_text_message(self, room_id, message): handler = self.hs.get_handlers().message_handler - event = yield handler.create_and_send_nonmember_event({ - "type": "m.room.message", - "content": {"body": "message", "msgtype": "m.text"}, - "room_id": room_id, - "sender": self.user.to_string(), - }) + event = yield handler.create_and_send_nonmember_event( + requester_for_user(self.user), + { + "type": "m.room.message", + "content": {"body": "message", "msgtype": "m.text"}, + "room_id": room_id, + "sender": self.user.to_string(), + } + ) defer.returnValue(event.event_id) @defer.inlineCallbacks diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 0785965de..1d210f9bf 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -86,7 +86,7 @@ class ProfileTestCase(unittest.TestCase): self.assertEquals(200, code) self.assertEquals(mocked_set.call_args[0][0].localpart, "1234ABCD") - self.assertEquals(mocked_set.call_args[0][1].localpart, "1234ABCD") + self.assertEquals(mocked_set.call_args[0][1].user.localpart, "1234ABCD") self.assertEquals(mocked_set.call_args[0][2], "Frank Jr.") @defer.inlineCallbacks @@ -155,5 +155,5 @@ class ProfileTestCase(unittest.TestCase): self.assertEquals(200, code) self.assertEquals(mocked_set.call_args[0][0].localpart, "1234ABCD") - self.assertEquals(mocked_set.call_args[0][1].localpart, "1234ABCD") + self.assertEquals(mocked_set.call_args[0][1].user.localpart, "1234ABCD") self.assertEquals(mocked_set.call_args[0][2], "http://my.server/pic.gif") diff --git a/tests/utils.py b/tests/utils.py index c67fa1ca3..291b54905 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -20,6 +20,7 @@ from synapse.storage.prepare_database import prepare_database from synapse.storage.engines import create_engine from synapse.server import HomeServer from synapse.federation.transport import server +from synapse.types import Requester from synapse.util.ratelimitutils import FederationRateLimiter from synapse.util.logcontext import LoggingContext @@ -510,3 +511,7 @@ class DeferredMockCallable(object): "call(%s)" % _format_call(c[0], c[1]) for c in calls ]) ) + + +def requester_for_user(user): + return Requester(user, None, False) From 5d6fbc1655dd25739563f9d554255b6282440164 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 3 Mar 2016 19:04:11 +0000 Subject: [PATCH 27/31] Empty commit From 62d808beccaf4ef94e743c57b9570bea99071eb7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 3 Mar 2016 11:38:36 +0000 Subject: [PATCH 28/31] jenkins-*.sh: set -x Also move the options from the shebang line to the body of the script, so that they take effect even if somebody explicitly runs "bash jenkins.sh" --- jenkins-flake8.sh | 4 +++- jenkins-postgres.sh | 4 +++- jenkins-sqlite.sh | 4 +++- jenkins-unittests.sh | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/jenkins-flake8.sh b/jenkins-flake8.sh index cbcb0ae4c..11f1cab6c 100755 --- a/jenkins-flake8.sh +++ b/jenkins-flake8.sh @@ -1,4 +1,6 @@ -#!/bin/bash -eu +#!/bin/bash + +set -eux : ${WORKSPACE:="$(pwd)"} diff --git a/jenkins-postgres.sh b/jenkins-postgres.sh index 1708cbfaa..d1fed590a 100755 --- a/jenkins-postgres.sh +++ b/jenkins-postgres.sh @@ -1,4 +1,6 @@ -#!/bin/bash -eu +#!/bin/bash + +set -eux : ${WORKSPACE:="$(pwd)"} diff --git a/jenkins-sqlite.sh b/jenkins-sqlite.sh index 2d98a0af9..57fd8de54 100755 --- a/jenkins-sqlite.sh +++ b/jenkins-sqlite.sh @@ -1,4 +1,6 @@ -#!/bin/bash -eu +#!/bin/bash + +set -eux : ${WORKSPACE:="$(pwd)"} diff --git a/jenkins-unittests.sh b/jenkins-unittests.sh index 2fa2f1b1d..104d51199 100755 --- a/jenkins-unittests.sh +++ b/jenkins-unittests.sh @@ -1,4 +1,6 @@ -#!/bin/bash -eu +#!/bin/bash + +set -eux : ${WORKSPACE:="$(pwd)"} From a92b4ea76fab9475483f345d998206d00f64b20f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 4 Mar 2016 00:06:03 +0000 Subject: [PATCH 29/31] Make sure we add all invited members before returning from createRoom add a missing yield. --- synapse/handlers/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 91fe306cf..2fb417b0c 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -210,7 +210,7 @@ class RoomCreationHandler(BaseHandler): ratelimit=False) for invitee in invite_list: - room_member_handler.update_membership( + yield room_member_handler.update_membership( requester, UserID.from_string(invitee), room_id, From 5fc59f009cca676ed8c9c932abf6cddc6614eae9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 23 Feb 2016 14:22:07 +0100 Subject: [PATCH 30/31] config,handlers/_base: added homeserver config for what state is included in a room invite Signed-off-by: Patrik Oldsberg --- synapse/config/api.py | 40 ++++++++++++++++++++++++++++++++++++ synapse/config/homeserver.py | 3 ++- synapse/handlers/_base.py | 8 ++------ 3 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 synapse/config/api.py diff --git a/synapse/config/api.py b/synapse/config/api.py new file mode 100644 index 000000000..20ba33226 --- /dev/null +++ b/synapse/config/api.py @@ -0,0 +1,40 @@ +# Copyright 2015, 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + +from synapse.api.constants import EventTypes + + +class ApiConfig(Config): + + def read_config(self, config): + self.room_invite_state_types = config.get("room_invite_state_types", [ + EventTypes.JoinRules, + EventTypes.CanonicalAlias, + EventTypes.RoomAvatar, + EventTypes.Name, + ]) + + def default_config(cls, **kwargs): + return """\ + ## API Configuration ## + + # A list of event types that will be included in the room_invite_state + room_invite_state_types: + - "{JoinRules}" + - "{CanonicalAlias}" + - "{RoomAvatar}" + - "{Name}" + """.format(**vars(EventTypes)) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 3c333b417..a08c170f1 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -23,6 +23,7 @@ from .captcha import CaptchaConfig from .voip import VoipConfig from .registration import RegistrationConfig from .metrics import MetricsConfig +from .api import ApiConfig from .appservice import AppServiceConfig from .key import KeyConfig from .saml2 import SAML2Config @@ -32,7 +33,7 @@ from .password import PasswordConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, - VoipConfig, RegistrationConfig, MetricsConfig, + VoipConfig, RegistrationConfig, MetricsConfig, ApiConfig, AppServiceConfig, KeyConfig, SAML2Config, CasConfig, PasswordConfig,): pass diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index c6a74b0e3..884572df9 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -333,12 +333,8 @@ class BaseHandler(object): "sender": e.sender, } for k, e in context.current_state.items() - if e.type in ( - EventTypes.JoinRules, - EventTypes.CanonicalAlias, - EventTypes.RoomAvatar, - EventTypes.Name, - ) or is_inviter_member_event(e) + if e.type in self.hs.config.room_invite_state_types + or is_inviter_member_event(e) ] invitee = UserID.from_string(event.state_key) From bb0e82fff186b509e82184ba679b7b8eb6db26c5 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 23 Feb 2016 14:48:12 +0100 Subject: [PATCH 31/31] tests/utils: added room_invite_state_types to test config Signed-off-by: Patrik Oldsberg --- tests/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/utils.py b/tests/utils.py index 291b54905..52405502e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -51,6 +51,7 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): config.macaroon_secret_key = "not even a little secret" config.server_name = "server.under.test" config.trusted_third_party_id_servers = [] + config.room_invite_state_types = [] config.database_config = {"name": "sqlite3"}