From 522f285f9bdffd2248b1eace2745db88d494eb85 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 1 Jun 2015 13:36:30 +0100 Subject: [PATCH 001/180] Add config option to disable compression of http responses --- synapse/app/homeserver.py | 10 ++++++++-- synapse/config/server.py | 6 ++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index d93afdc1c..65a5dfa84 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -87,10 +87,16 @@ class SynapseHomeServer(HomeServer): return MatrixFederationHttpClient(self) def build_resource_for_client(self): - return gz_wrap(ClientV1RestResource(self)) + res = ClientV1RestResource(self) + if self.config.gzip_responses: + res = gz_wrap(res) + return res def build_resource_for_client_v2_alpha(self): - return gz_wrap(ClientV2AlphaRestResource(self)) + res = ClientV2AlphaRestResource(self) + if self.config.gzip_responses: + res = gz_wrap(res) + return res def build_resource_for_federation(self): return JsonResource(self) diff --git a/synapse/config/server.py b/synapse/config/server.py index 48a26c65d..d0c8fb8f3 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -29,6 +29,7 @@ class ServerConfig(Config): self.soft_file_limit = config["soft_file_limit"] self.daemonize = config.get("daemonize") self.use_frozen_dicts = config.get("use_frozen_dicts", True) + self.gzip_responses = config["gzip_responses"] # Attempt to guess the content_addr for the v0 content repostitory content_addr = config.get("content_addr") @@ -86,6 +87,11 @@ class ServerConfig(Config): # Turn on the twisted telnet manhole service on localhost on the given # port. #manhole: 9000 + + # Should synapse compress HTTP responses to clients that support it? + # This should be disabled if running synapse behind a load balancer + # that can do automatic compression. + gzip_responses: True """ % locals() def read_arguments(self, args): From 6e174632289330aa494b15c3814743daebbf2127 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 1 Jun 2015 16:38:55 +0100 Subject: [PATCH 002/180] Don't explode if we don't have the event --- synapse/storage/events.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 20a8d8179..2caf0aae8 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -736,7 +736,8 @@ class EventsStore(SQLBaseStore): because = yield self.get_event( redaction_id, - check_redacted=False + check_redacted=False, + allow_none=True, ) if because: @@ -746,6 +747,7 @@ class EventsStore(SQLBaseStore): prev = yield self.get_event( ev.unsigned["replaces_state"], get_prev_content=False, + allow_none=True, ) if prev: ev.unsigned["prev_content"] = prev.get_dict()["content"] From 4bbfbf898ecfe9bd77b02a7e014d94163d1fd534 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 1 Jun 2015 17:02:23 +0100 Subject: [PATCH 003/180] Correctly pass in auth_events --- synapse/handlers/federation.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 46ce3699d..caf6a1f8e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -262,7 +262,13 @@ class FederationHandler(BaseHandler): yield defer.gatherResults( [ - self._handle_new_event(dest, a) + self._handle_new_event( + dest, a, + auth_events={ + (e.type, e.state_key): e for e in auth_events + if e.event_id in [a_id for a_id, _ in a.auth_events] + } + ) for a in auth_events.values() ], consumeErrors=True, @@ -274,6 +280,10 @@ class FederationHandler(BaseHandler): dest, event_map[e_id], state=events_to_state[e_id], backfilled=True, + auth_events={ + (e.type, e.state_key): e for e in auth_events + if e.event_id in [a_id for a_id, _ in a.auth_events] + } ) for e_id in events_to_state ], From 3f04a08a0c922962e3a5439b1740424202dc4616 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2015 10:11:32 +0100 Subject: [PATCH 004/180] Don't process events we've already processed. Remember to process state events --- synapse/handlers/federation.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index caf6a1f8e..acbb53d6c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -250,6 +250,7 @@ class FederationHandler(BaseHandler): # For each edge get the current state. auth_events = {} + state_events = {} events_to_state = {} for e_id in edges: state, auth = yield self.replication_layer.get_state_for_room( @@ -258,8 +259,13 @@ class FederationHandler(BaseHandler): event_id=e_id ) auth_events.update({a.event_id: a for a in auth}) + state_events.update({s.event_id: s for s in state}) events_to_state[e_id] = state + seen_events = yield self.store.have_events( + set(auth_events.keys()) | set(state_events.keys()) + ) + yield defer.gatherResults( [ self._handle_new_event( @@ -270,6 +276,22 @@ class FederationHandler(BaseHandler): } ) for a in auth_events.values() + if a.event_id not in seen_events + ], + consumeErrors=True, + ).addErrback(unwrapFirstError) + + yield defer.gatherResults( + [ + self._handle_new_event( + dest, s, + auth_events={ + (e.type, e.state_key): e for e in auth_events + if e.event_id in [a_id for a_id, _ in s.auth_events] + } + ) + for s in state_events.values() + if s.event_id not in seen_events ], consumeErrors=True, ).addErrback(unwrapFirstError) From fde0da6f19aeb6dee26fc0b89fcee9d27a50b8f4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2015 10:19:38 +0100 Subject: [PATCH 005/180] Correctly look up auth_events --- synapse/handlers/federation.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index acbb53d6c..5cd853d85 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -271,9 +271,10 @@ class FederationHandler(BaseHandler): self._handle_new_event( dest, a, auth_events={ - (e.type, e.state_key): e for e in auth_events - if e.event_id in [a_id for a_id, _ in a.auth_events] - } + (auth_events[a_id].type, auth_events[a_id].state_key): + auth_events[a_id] + for a_id, _ in a.auth_events + }, ) for a in auth_events.values() if a.event_id not in seen_events @@ -286,9 +287,10 @@ class FederationHandler(BaseHandler): self._handle_new_event( dest, s, auth_events={ - (e.type, e.state_key): e for e in auth_events - if e.event_id in [a_id for a_id, _ in s.auth_events] - } + (auth_events[a_id].type, auth_events[a_id].state_key): + auth_events[a_id] + for a_id, _ in s.auth_events + }, ) for s in state_events.values() if s.event_id not in seen_events @@ -303,9 +305,10 @@ class FederationHandler(BaseHandler): state=events_to_state[e_id], backfilled=True, auth_events={ - (e.type, e.state_key): e for e in auth_events - if e.event_id in [a_id for a_id, _ in a.auth_events] - } + (auth_events[a_id].type, auth_events[a_id].state_key): + auth_events[a_id] + for a_id, _ in event_map[e_id].auth_events + }, ) for e_id in events_to_state ], From e552b78d50a2f162a536057af48c7815fb181c30 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2015 10:28:14 +0100 Subject: [PATCH 006/180] Add some logging --- synapse/handlers/federation.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 5cd853d85..ced6fab16 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -247,6 +247,11 @@ class FederationHandler(BaseHandler): if set(e_id for e_id, _ in ev.prev_events) - event_ids ] + logger.info( + "backfill: Got %d events with %d edges", + len(events), len(edges), + ) + # For each edge get the current state. auth_events = {} From 02410e92395f53b88017bf93a82be41afe1a839e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2015 10:58:35 +0100 Subject: [PATCH 007/180] Handle the fact we might be missing auth events --- synapse/handlers/federation.py | 36 +++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ced6fab16..b33317534 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -264,6 +264,7 @@ class FederationHandler(BaseHandler): event_id=e_id ) auth_events.update({a.event_id: a for a in auth}) + auth_events.update({s.event_id: s for s in state}) state_events.update({s.event_id: s for s in state}) events_to_state[e_id] = state @@ -271,6 +272,25 @@ class FederationHandler(BaseHandler): set(auth_events.keys()) | set(state_events.keys()) ) + all_events = events + state_events.values() + auth_events.values() + required_auth = set( + a_id for event in all_events for a_id, _ in event.auth_events + ) + + missing_auth = required_auth - set(auth_events) + results = yield defer.gatherResults( + [ + self.replication_layer.get_pdu( + [dest], + event_id, + outlier=True, + ) + for event_id in missing_auth + ], + consumeErrors=True + ).addErrback(unwrapFirstError) + auth_events.update({a.event_id: a for a in results}) + yield defer.gatherResults( [ self._handle_new_event( @@ -287,22 +307,6 @@ class FederationHandler(BaseHandler): consumeErrors=True, ).addErrback(unwrapFirstError) - yield defer.gatherResults( - [ - self._handle_new_event( - dest, s, - auth_events={ - (auth_events[a_id].type, auth_events[a_id].state_key): - auth_events[a_id] - for a_id, _ in s.auth_events - }, - ) - for s in state_events.values() - if s.event_id not in seen_events - ], - consumeErrors=True, - ).addErrback(unwrapFirstError) - yield defer.gatherResults( [ self._handle_new_event( From 09e23334decf6e01533d2c15f49f418e9e8df6e5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2015 11:00:37 +0100 Subject: [PATCH 008/180] Add a timeout --- synapse/handlers/federation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b33317534..e8e3173ca 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -284,6 +284,7 @@ class FederationHandler(BaseHandler): [dest], event_id, outlier=True, + timeout=10000, ) for event_id in missing_auth ], From 5044e6c544ce98f4040052bf6b1b557ee31618bb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2015 15:39:08 +0100 Subject: [PATCH 009/180] Thumbnail images on a seperate thread --- synapse/rest/media/v1/base_resource.py | 96 ++++++++++++++------------ 1 file changed, 52 insertions(+), 44 deletions(-) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 4af5f7387..c8970165c 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -21,7 +21,7 @@ from synapse.api.errors import ( cs_error, Codes, SynapseError ) -from twisted.internet import defer +from twisted.internet import defer, threads from twisted.web.resource import Resource from twisted.protocols.basic import FileSender @@ -273,57 +273,65 @@ class BaseMediaResource(Resource): if not requirements: return + remote_thumbnails = [] + input_path = self.filepaths.remote_media_filepath(server_name, file_id) thumbnailer = Thumbnailer(input_path) m_width = thumbnailer.width m_height = thumbnailer.height - if m_width * m_height >= self.max_image_pixels: - logger.info( - "Image too large to thumbnail %r x %r > %r", - m_width, m_height, self.max_image_pixels - ) - return + def generate_thumbnails(): + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r", + m_width, m_height, self.max_image_pixels + ) + return - scales = set() - crops = set() - for r_width, r_height, r_method, r_type in requirements: - if r_method == "scale": - t_width, t_height = thumbnailer.aspect(r_width, r_height) - scales.add(( - min(m_width, t_width), min(m_height, t_height), r_type, - )) - elif r_method == "crop": - crops.add((r_width, r_height, r_type)) + scales = set() + crops = set() + for r_width, r_height, r_method, r_type in requirements: + if r_method == "scale": + t_width, t_height = thumbnailer.aspect(r_width, r_height) + scales.add(( + min(m_width, t_width), min(m_height, t_height), r_type, + )) + elif r_method == "crop": + crops.add((r_width, r_height, r_type)) - for t_width, t_height, t_type in scales: - t_method = "scale" - t_path = self.filepaths.remote_media_thumbnail( - server_name, file_id, t_width, t_height, t_type, t_method - ) - self._makedirs(t_path) - t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) - yield self.store.store_remote_media_thumbnail( - server_name, media_id, file_id, - t_width, t_height, t_type, t_method, t_len - ) + for t_width, t_height, t_type in scales: + t_method = "scale" + t_path = self.filepaths.remote_media_thumbnail( + server_name, file_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) + remote_thumbnails.append([ + server_name, media_id, file_id, + t_width, t_height, t_type, t_method, t_len + ]) - for t_width, t_height, t_type in crops: - if (t_width, t_height, t_type) in scales: - # If the aspect ratio of the cropped thumbnail matches a purely - # scaled one then there is no point in calculating a separate - # thumbnail. - continue - t_method = "crop" - t_path = self.filepaths.remote_media_thumbnail( - server_name, file_id, t_width, t_height, t_type, t_method - ) - self._makedirs(t_path) - t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) - yield self.store.store_remote_media_thumbnail( - server_name, media_id, file_id, - t_width, t_height, t_type, t_method, t_len - ) + for t_width, t_height, t_type in crops: + if (t_width, t_height, t_type) in scales: + # If the aspect ratio of the cropped thumbnail matches a purely + # scaled one then there is no point in calculating a separate + # thumbnail. + continue + t_method = "crop" + t_path = self.filepaths.remote_media_thumbnail( + server_name, file_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) + remote_thumbnails.append([ + server_name, media_id, file_id, + t_width, t_height, t_type, t_method, t_len + ]) + + yield threads.deferToThread(generate_thumbnails) + + for r in remote_thumbnails: + yield self.store.store_remote_media_thumbnail(*r) defer.returnValue({ "width": m_width, From 22716774d587354229149c752ace3b581013a077 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2015 14:26:54 +0100 Subject: [PATCH 010/180] Don't about JSON when warning about content tampering --- synapse/federation/federation_base.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index f0430b2cb..6799ed05a 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -18,8 +18,6 @@ from twisted.internet import defer from synapse.events.utils import prune_event -from syutil.jsonutil import encode_canonical_json - from synapse.crypto.event_signing import check_event_content_hash from synapse.api.errors import SynapseError @@ -120,16 +118,15 @@ class FederationBase(object): ) except SynapseError: logger.warn( - "Signature check failed for %s redacted to %s", - encode_canonical_json(pdu.get_pdu_json()), - encode_canonical_json(redacted_pdu_json), + "Signature check failed for %s, redacting", + pdu.event_id, ) raise if not check_event_content_hash(pdu): logger.warn( - "Event content has been tampered, redacting %s, %s", - pdu.event_id, encode_canonical_json(pdu.get_dict()) + "Event content has been tampered, redacting.", + pdu.event_id, ) defer.returnValue(redacted_event) From d3ded420b11f98fa9b3635099808643992f65e93 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2015 14:39:15 +0100 Subject: [PATCH 011/180] Rephrase log line --- synapse/federation/federation_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 6799ed05a..299493af9 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -118,7 +118,7 @@ class FederationBase(object): ) except SynapseError: logger.warn( - "Signature check failed for %s, redacting", + "Signature check failed for %s", pdu.event_id, ) raise From 3483b78d1a1fc01f36e39bddfb94a639e11bb126 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Jun 2015 18:15:13 +0100 Subject: [PATCH 012/180] Log where a request came from in federation --- synapse/federation/transport/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index af87805f3..31190e700 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -93,6 +93,8 @@ class TransportLayerServer(object): yield self.keyring.verify_json_for_server(origin, json_request) + logger.info("Request from %s", origin) + defer.returnValue((origin, content)) @log_function From 2ef2f6d593693351f399688c425add7a804f28d6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Jun 2015 10:15:27 +0100 Subject: [PATCH 013/180] SYN-403: Make content repository use its own http client. --- synapse/rest/media/v1/base_resource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 4af5f7387..9a60b777c 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -15,6 +15,7 @@ from .thumbnailer import Thumbnailer +from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.http.server import respond_with_json from synapse.util.stringutils import random_string from synapse.api.errors import ( @@ -52,7 +53,7 @@ class BaseMediaResource(Resource): def __init__(self, hs, filepaths): Resource.__init__(self) self.auth = hs.get_auth() - self.client = hs.get_http_client() + self.client = MatrixFederationHttpClient(hs) self.clock = hs.get_clock() self.server_name = hs.hostname self.store = hs.get_datastore() From d8866d72771fa04bd58a77a03aded45bf3ff2293 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Jun 2015 14:45:17 +0100 Subject: [PATCH 014/180] Caches should be bound to instances. Before, caches were global and so different instances of the stores would share caches. This caused problems in the unit tests. --- synapse/storage/_base.py | 45 +++++++++++----- tests/storage/test__base.py | 84 +++++++++++++++++------------- tests/storage/test_registration.py | 2 +- 3 files changed, 81 insertions(+), 50 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 39884c2af..8d33def6c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -127,7 +127,7 @@ class Cache(object): self.cache.clear() -def cached(max_entries=1000, num_args=1, lru=False): +class CacheDescriptor(object): """ A method decorator that applies a memoizing cache around the function. The function is presumed to take zero or more arguments, which are used in @@ -141,25 +141,32 @@ def cached(max_entries=1000, num_args=1, lru=False): which can be used to insert values into the cache specifically, without calling the calculation function. """ - def wrap(orig): + def __init__(self, orig, max_entries=1000, num_args=1, lru=False): + self.orig = orig + + self.max_entries = max_entries + self.num_args = num_args + self.lru = lru + + def __get__(self, obj, objtype=None): cache = Cache( - name=orig.__name__, - max_entries=max_entries, - keylen=num_args, - lru=lru, + name=self.orig.__name__, + max_entries=self.max_entries, + keylen=self.num_args, + lru=self.lru, ) - @functools.wraps(orig) + @functools.wraps(self.orig) @defer.inlineCallbacks - def wrapped(self, *keyargs): + def wrapped(*keyargs): try: - cached_result = cache.get(*keyargs) + cached_result = cache.get(*keyargs[:self.num_args]) if DEBUG_CACHES: - actual_result = yield orig(self, *keyargs) + actual_result = yield self.orig(obj, *keyargs) if actual_result != cached_result: logger.error( "Stale cache entry %s%r: cached: %r, actual %r", - orig.__name__, keyargs, + self.orig.__name__, keyargs, cached_result, actual_result, ) raise ValueError("Stale cache entry") @@ -170,18 +177,28 @@ def cached(max_entries=1000, num_args=1, lru=False): # while the SELECT is executing (SYN-369) sequence = cache.sequence - ret = yield orig(self, *keyargs) + ret = yield self.orig(obj, *keyargs) - cache.update(sequence, *keyargs + (ret,)) + cache.update(sequence, *keyargs[:self.num_args] + (ret,)) defer.returnValue(ret) wrapped.invalidate = cache.invalidate wrapped.invalidate_all = cache.invalidate_all wrapped.prefill = cache.prefill + + obj.__dict__[self.orig.__name__] = wrapped + return wrapped - return wrap + +def cached(max_entries=1000, num_args=1, lru=False): + return lambda orig: CacheDescriptor( + orig, + max_entries=max_entries, + num_args=num_args, + lru=lru + ) class LoggingTransaction(object): diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index 96caf8c4c..8c3d2952b 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -96,73 +96,84 @@ class CacheDecoratorTestCase(unittest.TestCase): @defer.inlineCallbacks def test_passthrough(self): - @cached() - def func(self, key): - return key + class A(object): + @cached() + def func(self, key): + return key - self.assertEquals((yield func(self, "foo")), "foo") - self.assertEquals((yield func(self, "bar")), "bar") + a = A() + + self.assertEquals((yield a.func("foo")), "foo") + self.assertEquals((yield a.func("bar")), "bar") @defer.inlineCallbacks def test_hit(self): callcount = [0] - @cached() - def func(self, key): - callcount[0] += 1 - return key + class A(object): + @cached() + def func(self, key): + callcount[0] += 1 + return key - yield func(self, "foo") + a = A() + yield a.func("foo") self.assertEquals(callcount[0], 1) - self.assertEquals((yield func(self, "foo")), "foo") + self.assertEquals((yield a.func("foo")), "foo") self.assertEquals(callcount[0], 1) @defer.inlineCallbacks def test_invalidate(self): callcount = [0] - @cached() - def func(self, key): - callcount[0] += 1 - return key + class A(object): + @cached() + def func(self, key): + callcount[0] += 1 + return key - yield func(self, "foo") + a = A() + yield a.func("foo") self.assertEquals(callcount[0], 1) - func.invalidate("foo") + a.func.invalidate("foo") - yield func(self, "foo") + yield a.func("foo") self.assertEquals(callcount[0], 2) def test_invalidate_missing(self): - @cached() - def func(self, key): - return key + class A(object): + @cached() + def func(self, key): + return key - func.invalidate("what") + A().func.invalidate("what") @defer.inlineCallbacks def test_max_entries(self): callcount = [0] - @cached(max_entries=10) - def func(self, key): - callcount[0] += 1 - return key + class A(object): + @cached(max_entries=10) + def func(self, key): + callcount[0] += 1 + return key - for k in range(0,12): - yield func(self, k) + a = A() + + for k in range(0, 12): + yield a.func(k) self.assertEquals(callcount[0], 12) # There must have been at least 2 evictions, meaning if we calculate # all 12 values again, we must get called at least 2 more times for k in range(0,12): - yield func(self, k) + yield a.func(k) self.assertTrue(callcount[0] >= 14, msg="Expected callcount >= 14, got %d" % (callcount[0])) @@ -171,12 +182,15 @@ class CacheDecoratorTestCase(unittest.TestCase): def test_prefill(self): callcount = [0] - @cached() - def func(self, key): - callcount[0] += 1 - return key + class A(object): + @cached() + def func(self, key): + callcount[0] += 1 + return key - func.prefill("foo", 123) + a = A() - self.assertEquals((yield func(self, "foo")), 123) + a.func.prefill("foo", 123) + + self.assertEquals((yield a.func("foo")), 123) self.assertEquals(callcount[0], 0) diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index 78f600420..270229117 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -46,7 +46,7 @@ class RegistrationStoreTestCase(unittest.TestCase): (yield self.store.get_user_by_id(self.user_id)) ) - result = yield self.store.get_user_by_token(self.tokens[1]) + result = yield self.store.get_user_by_token(self.tokens[0]) self.assertDictContainsSubset( { From 0d7d9c37b67bc9956524faf2a1b60c6fae07ae42 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Jun 2015 14:45:55 +0100 Subject: [PATCH 015/180] Add cache to get_state_groups --- synapse/storage/state.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index b24de34f2..f2b17f29e 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -81,19 +81,23 @@ class StateStore(SQLBaseStore): f, ) - @defer.inlineCallbacks - def c(vals): - vals[:] = yield self._get_events(vals, get_prev_content=False) - - yield defer.gatherResults( + state_list = yield defer.gatherResults( [ - c(vals) - for vals in states.values() + self._fetch_events_for_group(group, vals) + for group, vals in states.items() ], consumeErrors=True, ) - defer.returnValue(states) + defer.returnValue(dict(state_list)) + + @cached(num_args=1) + def _fetch_events_for_group(self, state_group, events): + return self._get_events( + events, get_prev_content=False + ).addCallback( + lambda evs: (state_group, evs) + ) def _store_state_groups_txn(self, txn, event, context): if context.current_state is None: From 1c3d844e7314dd5c1722ed77daf4bad8a056217d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Jun 2015 16:30:01 +0100 Subject: [PATCH 016/180] Don't needlessly compute context --- synapse/handlers/federation.py | 6 ++++-- synapse/state.py | 15 +++++++++++++-- tests/handlers/test_federation.py | 4 ++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 46ce3699d..5503d9ae8 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -900,8 +900,10 @@ class FederationHandler(BaseHandler): event.event_id, event.signatures, ) + outlier = event.internal_metadata.is_outlier() + context = yield self.state_handler.compute_event_context( - event, old_state=state + event, old_state=state, outlier=outlier, ) if not auth_events: @@ -912,7 +914,7 @@ class FederationHandler(BaseHandler): event.event_id, auth_events, ) - is_new_state = not event.internal_metadata.is_outlier() + is_new_state = not outlier # This is a hack to fix some old rooms where the initial join event # didn't reference the create event in its auth events. diff --git a/synapse/state.py b/synapse/state.py index 9dddb77d5..c1ce46d1b 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -106,7 +106,7 @@ class StateHandler(object): defer.returnValue(state) @defer.inlineCallbacks - def compute_event_context(self, event, old_state=None): + def compute_event_context(self, event, old_state=None, outlier=False): """ Fills out the context with the `current state` of the graph. The `current state` here is defined to be the state of the event graph just before the event - i.e. it never includes `event` @@ -119,9 +119,20 @@ class StateHandler(object): Returns: an EventContext """ + yield run_on_reactor() + context = EventContext() - yield run_on_reactor() + if outlier: + if old_state: + context.current_state = { + (s.type, s.state_key): s for s in old_state + } + else: + context.current_state = {} + context.prev_state_events = [] + context.state_group = None + defer.returnValue(context) if old_state: context.current_state = { diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index f3821242b..d392c2301 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -100,7 +100,7 @@ class FederationTestCase(unittest.TestCase): return defer.succeed({}) self.datastore.have_events.side_effect = have_events - def annotate(ev, old_state=None): + def annotate(ev, old_state=None, outlier=False): context = Mock() context.current_state = {} context.auth_events = {} @@ -120,7 +120,7 @@ class FederationTestCase(unittest.TestCase): ) self.state_handler.compute_event_context.assert_called_once_with( - ANY, old_state=None, + ANY, old_state=None, outlier=False ) self.auth.check.assert_called_once_with(ANY, auth_events={}) From 53f0bf85d7781960d61ea0d4e3f2435102ca7fb6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Jun 2015 16:43:40 +0100 Subject: [PATCH 017/180] Comment --- synapse/state.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/state.py b/synapse/state.py index c1ce46d1b..8a635f1cc 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -124,6 +124,9 @@ class StateHandler(object): context = EventContext() if outlier: + # If this is an outlier, then we know it shouldn't have any current + # state. Certainly store.get_current_state won't return any, and + # persisting the event won't store the state group. if old_state: context.current_state = { (s.type, s.state_key): s for s in old_state From 55bf90b9e458c6adbbd0e82403f9da06da95a812 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Jun 2015 16:06:39 +0100 Subject: [PATCH 018/180] Don't needlessly compute prev_state --- synapse/handlers/_base.py | 4 +++- synapse/state.py | 4 ---- tests/handlers/test_room.py | 2 ++ 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 833ff4137..d6c064b39 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -78,7 +78,9 @@ class BaseHandler(object): context = yield state_handler.compute_event_context(builder) if builder.is_state(): - builder.prev_state = context.prev_state_events + builder.prev_state = yield self.store.add_event_hashes( + context.prev_state_events + ) yield self.auth.add_auth_events(builder, context) diff --git a/synapse/state.py b/synapse/state.py index 9dddb77d5..d4d893000 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -155,10 +155,6 @@ class StateHandler(object): context.current_state = curr_state context.state_group = group if not event.is_state() else None - prev_state = yield self.store.add_event_hashes( - prev_state - ) - if event.is_state(): key = (event.type, event.state_key) if key in context.current_state: diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index a2d763599..2a7553f98 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -42,6 +42,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "get_room", "store_room", "get_latest_events_in_room", + "add_event_hashes", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), @@ -88,6 +89,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.ratelimiter.send_message.return_value = (True, 0) self.datastore.persist_event.return_value = (1,1) + self.datastore.add_event_hashes.return_value = [] @defer.inlineCallbacks def test_invite(self): From 63a7b3ad1eb6f5e959ead1bae36d9037cad888a2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 4 Jun 2015 16:16:01 +0100 Subject: [PATCH 019/180] Add script to (re)convert the pushers table to changing the unique key. Also give the python db upgrade scripts the database engine so they can convert parameter strings, and add *args **kwargs to the upgrade function so we can add more args in future and previous scripts will ignore them. --- synapse/storage/__init__.py | 2 +- .../schema/delta/14/upgrade_appservice_db.py | 2 +- synapse/storage/schema/delta/20/pushers.py | 77 +++++++++++++++++++ 3 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 synapse/storage/schema/delta/20/pushers.py diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 75af44d78..71757c393 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -348,7 +348,7 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, module_name, absolute_path, python_file ) logger.debug("Running script %s", relative_path) - module.run_upgrade(cur) + module.run_upgrade(cur, database_engine) elif ext == ".sql": # A plain old .sql file, just read and execute it logger.debug("Applying schema %s", relative_path) diff --git a/synapse/storage/schema/delta/14/upgrade_appservice_db.py b/synapse/storage/schema/delta/14/upgrade_appservice_db.py index 9f3a4dd4c..61232f975 100644 --- a/synapse/storage/schema/delta/14/upgrade_appservice_db.py +++ b/synapse/storage/schema/delta/14/upgrade_appservice_db.py @@ -18,7 +18,7 @@ import logging logger = logging.getLogger(__name__) -def run_upgrade(cur): +def run_upgrade(cur, *args, **kwargs): cur.execute("SELECT id, regex FROM application_services_regex") for row in cur.fetchall(): try: diff --git a/synapse/storage/schema/delta/20/pushers.py b/synapse/storage/schema/delta/20/pushers.py new file mode 100644 index 000000000..f09b41559 --- /dev/null +++ b/synapse/storage/schema/delta/20/pushers.py @@ -0,0 +1,77 @@ +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Main purpose of this upgrade is to change the unique key on the +pushers table again (it was missed when the v16 full schema was +made) but this also changes the pushkey and data columns to text. +When selecting a bytea column into a text column, postgres inserts +the hex encoded data, and there's no portable way of getting the +UTF-8 bytes, so we have to do it in Python. +""" + +import logging + +logger = logging.getLogger(__name__) + + +def run_upgrade(cur, database_engine, *args, **kwargs): + logger.info("Porting pushers table...") + cur.execute(""" + CREATE TABLE IF NOT EXISTS pushers2 ( + id BIGINT PRIMARY KEY, + user_name TEXT NOT NULL, + access_token BIGINT DEFAULT NULL, + profile_tag VARCHAR(32) NOT NULL, + kind VARCHAR(8) NOT NULL, + app_id VARCHAR(64) NOT NULL, + app_display_name VARCHAR(64) NOT NULL, + device_display_name VARCHAR(128) NOT NULL, + pushkey TEXT NOT NULL, + ts BIGINT NOT NULL, + lang VARCHAR(8), + data TEXT, + last_token TEXT, + last_success BIGINT, + failing_since BIGINT, + UNIQUE (app_id, pushkey, user_name) + ) + """) + cur.execute("""SELECT + id, user_name, access_token, profile_tag, kind, + app_id, app_display_name, device_display_name, + pushkey, ts, lang, data, last_token, last_success, + failing_since + FROM pushers + """) + count = 0 + for row in cur.fetchall(): + row = list(row) + row[8] = bytes(row[8]).decode("utf-8") + row[11] = bytes(row[11]).decode("utf-8") + cur.execute(database_engine.convert_param_style(""" + INSERT into pushers2 ( + id, user_name, access_token, profile_tag, kind, + app_id, app_display_name, device_display_name, + pushkey, ts, lang, data, last_token, last_success, + failing_since + ) values (%s)""" % (','.join(['?' for _ in range(len(row))]))), + row + ) + count += 1 + cur.execute("DROP TABLE pushers") + cur.execute("ALTER TABLE pushers2 RENAME TO pushers") + logger.info("Moved %d pushers to new table", count) + From da84946de416aff67eea67550cc2b68c95d2868d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 4 Jun 2015 16:43:45 +0100 Subject: [PATCH 020/180] pep8 --- synapse/storage/schema/delta/20/pushers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/schema/delta/20/pushers.py b/synapse/storage/schema/delta/20/pushers.py index f09b41559..543e57bbe 100644 --- a/synapse/storage/schema/delta/20/pushers.py +++ b/synapse/storage/schema/delta/20/pushers.py @@ -74,4 +74,3 @@ def run_upgrade(cur, database_engine, *args, **kwargs): cur.execute("DROP TABLE pushers") cur.execute("ALTER TABLE pushers2 RENAME TO pushers") logger.info("Moved %d pushers to new table", count) - From 378a0f7a79a22a1211e3d048b7fd9e99d1efc7de Mon Sep 17 00:00:00 2001 From: manuroe Date: Thu, 4 Jun 2015 17:58:17 +0200 Subject: [PATCH 021/180] Federation demo start.sh: Fixed --no-rate-limit param in the script --- demo/clean.sh | 6 ++++-- demo/start.sh | 17 +++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/demo/clean.sh b/demo/clean.sh index c5dabd476..31007737f 100755 --- a/demo/clean.sh +++ b/demo/clean.sh @@ -11,7 +11,9 @@ if [ -f $PID_FILE ]; then exit 1 fi -find "$DIR" -name "*.log" -delete -find "$DIR" -name "*.db" -delete +for port in 8080 8081 8082; do + rm -rf demo/$port + rm -rf demo/media_store.$port +done rm -rf $DIR/etc diff --git a/demo/start.sh b/demo/start.sh index b9cc14b9d..b5dea5e17 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -8,14 +8,6 @@ cd "$DIR/.." mkdir -p demo/etc -# Check the --no-rate-limit param -PARAMS="" -if [ $# -eq 1 ]; then - if [ $1 = "--no-rate-limit" ]; then - PARAMS="--rc-messages-per-second 1000 --rc-message-burst-count 1000" - fi -fi - export PYTHONPATH=$(readlink -f $(pwd)) @@ -35,6 +27,15 @@ for port in 8080 8081 8082; do -H "localhost:$https_port" \ --config-path "$DIR/etc/$port.config" \ + # Check script parameters + if [ $# -eq 1 ]; then + if [ $1 = "--no-rate-limit" ]; then + # Set high limits in config file to disable rate limiting + perl -p -i -e 's/rc_messages_per_second.*/rc_messages_per_second: 1000/g' $DIR/etc/$port.config + perl -p -i -e 's/rc_message_burst_count.*/rc_message_burst_count: 1000/g' $DIR/etc/$port.config + fi + fi + python -m synapse.app.homeserver \ --config-path "$DIR/etc/$port.config" \ -D \ From b8690dd840ea293717cd44bffe91e2cfe36bd9c8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 5 Jun 2015 11:40:22 +0100 Subject: [PATCH 022/180] Catch any exceptions in the pusher loop. Use a lower timeout for pushers so we can see if they're actually still running. --- synapse/push/__init__.py | 257 +++++++++++++++++++++------------------ 1 file changed, 139 insertions(+), 118 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8059fff1b..36f450c31 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -24,6 +24,7 @@ import baserules import logging import simplejson as json import re +import random logger = logging.getLogger(__name__) @@ -256,134 +257,154 @@ class Pusher(object): logger.info("Pusher %s for user %s starting from token %s", self.pushkey, self.user_name, self.last_token) + wait = 0 while self.alive: - from_tok = StreamToken.from_string(self.last_token) - config = PaginationConfig(from_token=from_tok, limit='1') - chunk = yield self.evStreamHandler.get_stream( - self.user_name, config, - timeout=100*365*24*60*60*1000, affect_presence=False - ) - - # limiting to 1 may get 1 event plus 1 presence event, so - # pick out the actual event - single_event = None - for c in chunk['chunk']: - if 'event_id' in c: # Hmmm... - single_event = c - break - if not single_event: - self.last_token = chunk['end'] - continue - - if not self.alive: - continue - - processed = False - actions = yield self._actions_for_event(single_event) - tweaks = _tweaks_for_actions(actions) - - if len(actions) == 0: - logger.warn("Empty actions! Using default action.") - actions = Pusher.DEFAULT_ACTIONS - - if 'notify' not in actions and 'dont_notify' not in actions: - logger.warn("Neither notify nor dont_notify in actions: adding default") - actions.extend(Pusher.DEFAULT_ACTIONS) - - if 'dont_notify' in actions: - logger.debug( - "%s for %s: dont_notify", - single_event['event_id'], self.user_name + try: + if wait > 0: + yield synapse.util.async.sleep(wait) + yield self.get_and_dispatch() + wait = 0 + except: + if wait == 0: + wait = 1 + else: + wait = min(wait * 2, 1800) + logger.exception( + "Exception in pusher loop for pushkey %s. Pausing for %ds", + self.pushkey, wait ) + + @defer.inlineCallbacks + def get_and_dispatch(self): + from_tok = StreamToken.from_string(self.last_token) + config = PaginationConfig(from_token=from_tok, limit='1') + timeout = (300 + random.randint(-60, 60)) * 1000 + chunk = yield self.evStreamHandler.get_stream( + self.user_name, config, + timeout=timeout, affect_presence=False + ) + + # limiting to 1 may get 1 event plus 1 presence event, so + # pick out the actual event + single_event = None + for c in chunk['chunk']: + if 'event_id' in c: # Hmmm... + single_event = c + break + if not single_event: + self.last_token = chunk['end'] + logger.debug("Event stream timeout for pushkey %s", self.pushkey) + return + + if not self.alive: + return + + processed = False + actions = yield self._actions_for_event(single_event) + tweaks = _tweaks_for_actions(actions) + + if len(actions) == 0: + logger.warn("Empty actions! Using default action.") + actions = Pusher.DEFAULT_ACTIONS + + if 'notify' not in actions and 'dont_notify' not in actions: + logger.warn("Neither notify nor dont_notify in actions: adding default") + actions.extend(Pusher.DEFAULT_ACTIONS) + + if 'dont_notify' in actions: + logger.debug( + "%s for %s: dont_notify", + single_event['event_id'], self.user_name + ) + processed = True + else: + rejected = yield self.dispatch_push(single_event, tweaks) + self.has_unread = True + if isinstance(rejected, list) or isinstance(rejected, tuple): processed = True - else: - rejected = yield self.dispatch_push(single_event, tweaks) - self.has_unread = True - if isinstance(rejected, list) or isinstance(rejected, tuple): - processed = True - for pk in rejected: - if pk != self.pushkey: - # for sanity, we only remove the pushkey if it - # was the one we actually sent... - logger.warn( - ("Ignoring rejected pushkey %s because we" - " didn't send it"), pk - ) - else: - logger.info( - "Pushkey %s was rejected: removing", - pk - ) - yield self.hs.get_pusherpool().remove_pusher( - self.app_id, pk, self.user_name - ) + for pk in rejected: + if pk != self.pushkey: + # for sanity, we only remove the pushkey if it + # was the one we actually sent... + logger.warn( + ("Ignoring rejected pushkey %s because we" + " didn't send it"), pk + ) + else: + logger.info( + "Pushkey %s was rejected: removing", + pk + ) + yield self.hs.get_pusherpool().remove_pusher( + self.app_id, pk, self.user_name + ) - if not self.alive: - continue + if not self.alive: + return - if processed: - self.backoff_delay = Pusher.INITIAL_BACKOFF - self.last_token = chunk['end'] - self.store.update_pusher_last_token_and_success( + if processed: + self.backoff_delay = Pusher.INITIAL_BACKOFF + self.last_token = chunk['end'] + self.store.update_pusher_last_token_and_success( + self.app_id, + self.pushkey, + self.user_name, + self.last_token, + self.clock.time_msec() + ) + if self.failing_since: + self.failing_since = None + self.store.update_pusher_failing_since( self.app_id, self.pushkey, self.user_name, - self.last_token, - self.clock.time_msec() + self.failing_since) + else: + if not self.failing_since: + self.failing_since = self.clock.time_msec() + self.store.update_pusher_failing_since( + self.app_id, + self.pushkey, + self.user_name, + self.failing_since + ) + + if (self.failing_since and + self.failing_since < + self.clock.time_msec() - Pusher.GIVE_UP_AFTER): + # we really only give up so that if the URL gets + # fixed, we don't suddenly deliver a load + # of old notifications. + logger.warn("Giving up on a notification to user %s, " + "pushkey %s", + self.user_name, self.pushkey) + self.backoff_delay = Pusher.INITIAL_BACKOFF + self.last_token = chunk['end'] + self.store.update_pusher_last_token( + self.app_id, + self.pushkey, + self.user_name, + self.last_token + ) + + self.failing_since = None + self.store.update_pusher_failing_since( + self.app_id, + self.pushkey, + self.user_name, + self.failing_since ) - if self.failing_since: - self.failing_since = None - self.store.update_pusher_failing_since( - self.app_id, - self.pushkey, - self.user_name, - self.failing_since) else: - if not self.failing_since: - self.failing_since = self.clock.time_msec() - self.store.update_pusher_failing_since( - self.app_id, - self.pushkey, - self.user_name, - self.failing_since - ) - - if (self.failing_since and - self.failing_since < - self.clock.time_msec() - Pusher.GIVE_UP_AFTER): - # we really only give up so that if the URL gets - # fixed, we don't suddenly deliver a load - # of old notifications. - logger.warn("Giving up on a notification to user %s, " - "pushkey %s", - self.user_name, self.pushkey) - self.backoff_delay = Pusher.INITIAL_BACKOFF - self.last_token = chunk['end'] - self.store.update_pusher_last_token( - self.app_id, - self.pushkey, - self.user_name, - self.last_token - ) - - self.failing_since = None - self.store.update_pusher_failing_since( - self.app_id, - self.pushkey, - self.user_name, - self.failing_since - ) - else: - logger.warn("Failed to dispatch push for user %s " - "(failing for %dms)." - "Trying again in %dms", - self.user_name, - self.clock.time_msec() - self.failing_since, - self.backoff_delay) - yield synapse.util.async.sleep(self.backoff_delay / 1000.0) - self.backoff_delay *= 2 - if self.backoff_delay > Pusher.MAX_BACKOFF: - self.backoff_delay = Pusher.MAX_BACKOFF + logger.warn("Failed to dispatch push for user %s " + "(failing for %dms)." + "Trying again in %dms", + self.user_name, + self.clock.time_msec() - self.failing_since, + self.backoff_delay) + yield synapse.util.async.sleep(self.backoff_delay / 1000.0) + self.backoff_delay *= 2 + if self.backoff_delay > Pusher.MAX_BACKOFF: + self.backoff_delay = Pusher.MAX_BACKOFF def stop(self): self.alive = False From f7f07dc517941d628dcb362a7511712728912141 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 11 Jun 2015 15:48:52 +0100 Subject: [PATCH 023/180] Begin changing the config format --- synapse/config/server.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index d0c8fb8f3..022ebcea9 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -61,14 +61,6 @@ class ServerConfig(Config): # e.g. matrix.org, localhost:8080, etc. server_name: "%(server_name)s" - # The port to listen for HTTPS requests on. - # For when matrix traffic is sent directly to synapse. - bind_port: %(bind_port)s - - # The port to listen for HTTP requests on. - # For when matrix traffic passes through loadbalancer that unwraps TLS. - unsecure_port: %(unsecure_port)s - # Local interface to listen on. # The empty string will cause synapse to listen on all interfaces. bind_host: "" @@ -92,6 +84,30 @@ class ServerConfig(Config): # This should be disabled if running synapse behind a load balancer # that can do automatic compression. gzip_responses: True + + listeners: + # For when matrix traffic is sent directly to synapse. + secure: + # The type of + type: http_resource + + # The port to listen for HTTPS requests on. + port: %(bind_port)s + + # Is this a TLS socket? + tls: true + + # Local interface to listen on. + # The empty string will cause synapse to listen on all interfaces. + bind_address: "" + + # For when matrix traffic passes through loadbalancer that unwraps TLS. + unsecure: + port: %(unsecure_port)s + tls: false + bind_address: "" + + """ % locals() def read_arguments(self, args): From 2d41dc0069f62b475824811e2cb75067750b43a4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 11 Jun 2015 15:49:19 +0100 Subject: [PATCH 024/180] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 4720d9984..e6088dc6c 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.9.1" +__version__ = "0.9.2" From 1e365e88bd9496e60de56e45ea4603eb3b81d8c9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 11 Jun 2015 15:50:39 +0100 Subject: [PATCH 025/180] Bump schema version --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 71757c393..c137f4782 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 19 +SCHEMA_VERSION = 20 dir_path = os.path.abspath(os.path.dirname(__file__)) From 716cf144ec80f570da29373e3d6196940012a6ef Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 11 Jun 2015 16:07:06 +0100 Subject: [PATCH 026/180] Update change log --- CHANGES.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 1ca2407a7..778d3c00e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,26 @@ +Changes in synapse v0.9.2 (2015-06-12) +====================================== + +General: + +* Use ultrajson for json deserialisation, and for serialisation when a + canonical encoding is not required. Ultrajson is significantly faster + than simplejson in some circumstances. +* Use connection pools for outgoing HTTP connections. +* Process thumbnails on separate threads. + +Configuration: + +* Add option to disable HTTP response compression. + +Federation: + +* Improve resilience of backfill by ensuring we fetch any missing auth events. +* Improve performance of backfill and joining remote rooms by removing + unnecessary computations. This included handling events we'd previously + handled as well as attempting to compute the current state for outliers. + + Changes in synapse v0.9.1 (2015-05-26) ====================================== From 4dd47236e736ae201a3bde4d6048e37416cddc22 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 11:42:52 +0100 Subject: [PATCH 027/180] Update change log --- CHANGES.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 778d3c00e..fa35a51fb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,9 +3,9 @@ Changes in synapse v0.9.2 (2015-06-12) General: -* Use ultrajson for json deserialisation, and for serialisation when a - canonical encoding is not required. Ultrajson is significantly faster - than simplejson in some circumstances. +* Use ultrajson for json (de)serialisation when a canonical encoding is not + required. Ultrajson is significantly faster than simplejson in some + circumstances. * Use connection pools for outgoing HTTP connections. * Process thumbnails on separate threads. From a3dc31cab98a107d98ad014eb91e4a0b5dff1450 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 11:45:13 +0100 Subject: [PATCH 028/180] s/some/certain --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index fa35a51fb..b7b1fdfbd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,7 @@ Changes in synapse v0.9.2 (2015-06-12) General: * Use ultrajson for json (de)serialisation when a canonical encoding is not - required. Ultrajson is significantly faster than simplejson in some + required. Ultrajson is significantly faster than simplejson in certain circumstances. * Use connection pools for outgoing HTTP connections. * Process thumbnails on separate threads. From 1a87f5f26c6c54856bf53516ad4d820da45e146d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 11:46:41 +0100 Subject: [PATCH 029/180] Mention config option name --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b7b1fdfbd..ac398728f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,7 +11,7 @@ General: Configuration: -* Add option to disable HTTP response compression. +* Add option, ``gzip_responses``, to disable HTTP response compression. Federation: From c42ed47660e99026d50de833b0b06c75a3b25d37 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 11:52:52 +0100 Subject: [PATCH 030/180] Fix up create_resource_tree --- synapse/app/homeserver.py | 204 +++++++++++++++++++------------------- 1 file changed, 100 insertions(+), 104 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 65a5dfa84..79c7a8558 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -145,109 +145,6 @@ class SynapseHomeServer(HomeServer): **self.db_config.get("args", {}) ) - def create_resource_tree(self, redirect_root_to_web_client): - """Create the resource tree for this Home Server. - - This in unduly complicated because Twisted does not support putting - child resources more than 1 level deep at a time. - - Args: - web_client (bool): True to enable the web client. - redirect_root_to_web_client (bool): True to redirect '/' to the - location of the web client. This does nothing if web_client is not - True. - """ - config = self.get_config() - web_client = config.web_client - - # list containing (path_str, Resource) e.g: - # [ ("/aaa/bbb/cc", Resource1), ("/aaa/dummy", Resource2) ] - desired_tree = [ - (CLIENT_PREFIX, self.get_resource_for_client()), - (CLIENT_V2_ALPHA_PREFIX, self.get_resource_for_client_v2_alpha()), - (FEDERATION_PREFIX, self.get_resource_for_federation()), - (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), - (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), - (SERVER_KEY_V2_PREFIX, self.get_resource_for_server_key_v2()), - (MEDIA_PREFIX, self.get_resource_for_media_repository()), - (STATIC_PREFIX, self.get_resource_for_static_content()), - ] - - if web_client: - logger.info("Adding the web client.") - desired_tree.append((WEB_CLIENT_PREFIX, - self.get_resource_for_web_client())) - - if web_client and redirect_root_to_web_client: - self.root_resource = RootRedirect(WEB_CLIENT_PREFIX) - else: - self.root_resource = Resource() - - metrics_resource = self.get_resource_for_metrics() - if config.metrics_port is None and metrics_resource is not None: - desired_tree.append((METRICS_PREFIX, metrics_resource)) - - # ideally we'd just use getChild and putChild but getChild doesn't work - # unless you give it a Request object IN ADDITION to the name :/ So - # instead, we'll store a copy of this mapping so we can actually add - # extra resources to existing nodes. See self._resource_id for the key. - resource_mappings = {} - for full_path, res in desired_tree: - logger.info("Attaching %s to path %s", res, full_path) - last_resource = self.root_resource - for path_seg in full_path.split('/')[1:-1]: - if path_seg not in last_resource.listNames(): - # resource doesn't exist, so make a "dummy resource" - child_resource = Resource() - last_resource.putChild(path_seg, child_resource) - res_id = self._resource_id(last_resource, path_seg) - resource_mappings[res_id] = child_resource - last_resource = child_resource - else: - # we have an existing Resource, use that instead. - res_id = self._resource_id(last_resource, path_seg) - last_resource = resource_mappings[res_id] - - # =========================== - # now attach the actual desired resource - last_path_seg = full_path.split('/')[-1] - - # if there is already a resource here, thieve its children and - # replace it - res_id = self._resource_id(last_resource, last_path_seg) - if res_id in resource_mappings: - # there is a dummy resource at this path already, which needs - # to be replaced with the desired resource. - existing_dummy_resource = resource_mappings[res_id] - for child_name in existing_dummy_resource.listNames(): - child_res_id = self._resource_id(existing_dummy_resource, - child_name) - child_resource = resource_mappings[child_res_id] - # steal the children - res.putChild(child_name, child_resource) - - # finally, insert the desired resource in the right place - last_resource.putChild(last_path_seg, res) - res_id = self._resource_id(last_resource, last_path_seg) - resource_mappings[res_id] = res - - return self.root_resource - - def _resource_id(self, resource, path_seg): - """Construct an arbitrary resource ID so you can retrieve the mapping - later. - - If you want to represent resource A putChild resource B with path C, - the mapping should looks like _resource_id(A,C) = B. - - Args: - resource (Resource): The *parent* Resource - path_seg (str): The name of the child Resource to be attached. - Returns: - str: A unique string which can be a key to the child Resource. - """ - return "%s-%s" % (resource, path_seg) - def start_listening(self): config = self.get_config() @@ -447,7 +344,26 @@ def setup(config_options): database_engine=database_engine, ) - hs.create_resource_tree( + resources = { + CLIENT_PREFIX: hs.get_resource_for_client(), + CLIENT_V2_ALPHA_PREFIX: hs.get_resource_for_client_v2_alpha(), + FEDERATION_PREFIX: hs.get_resource_for_federation(), + CONTENT_REPO_PREFIX: hs.get_resource_for_content_repo(), + SERVER_KEY_PREFIX: hs.get_resource_for_server_key(), + SERVER_KEY_V2_PREFIX: hs.get_resource_for_server_key_v2(), + MEDIA_PREFIX: hs.get_resource_for_media_repository(), + STATIC_PREFIX: hs.get_resource_for_static_content(), + } + + if config.web_client: + resources[WEB_CLIENT_PREFIX] = hs.get_resource_for_web_client() + + metrics_resource = hs.get_resource_for_metrics() + if config.metrics_port is None and metrics_resource is not None: + resources[METRICS_PREFIX] = metrics_resource + + hs.root_resource = create_resource_tree( + resources, redirect_root_to_web_client=True, ) @@ -525,6 +441,86 @@ class SynapseSite(Site): self.access_logger.info(line) +def create_resource_tree(desired_tree, redirect_root_to_web_client=True): + """Create the resource tree for this Home Server. + + This in unduly complicated because Twisted does not support putting + child resources more than 1 level deep at a time. + + Args: + web_client (bool): True to enable the web client. + redirect_root_to_web_client (bool): True to redirect '/' to the + location of the web client. This does nothing if web_client is not + True. + """ + if redirect_root_to_web_client and WEB_CLIENT_PREFIX in desired_tree: + root_resource = RootRedirect(WEB_CLIENT_PREFIX) + else: + root_resource = Resource() + + # ideally we'd just use getChild and putChild but getChild doesn't work + # unless you give it a Request object IN ADDITION to the name :/ So + # instead, we'll store a copy of this mapping so we can actually add + # extra resources to existing nodes. See self._resource_id for the key. + resource_mappings = {} + for full_path, res in desired_tree.items(): + logger.info("Attaching %s to path %s", res, full_path) + last_resource = root_resource + for path_seg in full_path.split('/')[1:-1]: + if path_seg not in last_resource.listNames(): + # resource doesn't exist, so make a "dummy resource" + child_resource = Resource() + last_resource.putChild(path_seg, child_resource) + res_id = _resource_id(last_resource, path_seg) + resource_mappings[res_id] = child_resource + last_resource = child_resource + else: + # we have an existing Resource, use that instead. + res_id = _resource_id(last_resource, path_seg) + last_resource = resource_mappings[res_id] + + # =========================== + # now attach the actual desired resource + last_path_seg = full_path.split('/')[-1] + + # if there is already a resource here, thieve its children and + # replace it + res_id = _resource_id(last_resource, last_path_seg) + if res_id in resource_mappings: + # there is a dummy resource at this path already, which needs + # to be replaced with the desired resource. + existing_dummy_resource = resource_mappings[res_id] + for child_name in existing_dummy_resource.listNames(): + child_res_id = _resource_id(existing_dummy_resource, + child_name) + child_resource = resource_mappings[child_res_id] + # steal the children + res.putChild(child_name, child_resource) + + # finally, insert the desired resource in the right place + last_resource.putChild(last_path_seg, res) + res_id = _resource_id(last_resource, last_path_seg) + resource_mappings[res_id] = res + + return root_resource + + +def _resource_id(resource, path_seg): + """Construct an arbitrary resource ID so you can retrieve the mapping + later. + + If you want to represent resource A putChild resource B with path C, + the mapping should looks like _resource_id(A,C) = B. + + Args: + resource (Resource): The *parent* Resource + path_seg (str): The name of the child Resource to be attached. + Returns: + str: A unique string which can be a key to the child Resource. + """ + return "%s-%s" % (resource, path_seg) + + def run(hs): PROFILE_SYNAPSE = False if PROFILE_SYNAPSE: From fd2c07bfed9182f4fc3ccf50b4390645fe038da4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 15:33:07 +0100 Subject: [PATCH 031/180] Use config.listeners --- synapse/app/homeserver.py | 150 +++++++++++++++++++++----------------- synapse/config/server.py | 108 ++++++++++++++++++++------- 2 files changed, 165 insertions(+), 93 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 79c7a8558..4228bac67 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -87,16 +87,10 @@ class SynapseHomeServer(HomeServer): return MatrixFederationHttpClient(self) def build_resource_for_client(self): - res = ClientV1RestResource(self) - if self.config.gzip_responses: - res = gz_wrap(res) - return res + return ClientV1RestResource(self) def build_resource_for_client_v2_alpha(self): - res = ClientV2AlphaRestResource(self) - if self.config.gzip_responses: - res = gz_wrap(res) - return res + return ClientV2AlphaRestResource(self) def build_resource_for_federation(self): return JsonResource(self) @@ -145,49 +139,102 @@ class SynapseHomeServer(HomeServer): **self.db_config.get("args", {}) ) - def start_listening(self): - config = self.get_config() + def _listener_http(self, config, listener_config): + port = listener_config["port"] + bind_address = listener_config.get("bind_address", "") + tls = listener_config.get("tls", False) - if not config.no_tls and config.bind_port is not None: + if tls and config.no_tls: + return + + metrics_resource = self.get_resource_for_metrics() + + resources = {} + for res in listener_config["resources"]: + for name in res["names"]: + if name == "client": + if res["compress"]: + client_v1 = gz_wrap(self.get_resource_for_client()) + client_v2 = gz_wrap(self.get_resource_for_client_v2_alpha()) + else: + client_v1 = self.get_resource_for_client() + client_v2 = self.get_resource_for_client_v2_alpha() + + resources.update({ + CLIENT_PREFIX: client_v1, + CLIENT_V2_ALPHA_PREFIX: client_v2, + }) + + if name == "federation": + resources.update({ + FEDERATION_PREFIX: self.get_resource_for_federation(), + }) + + if name in ["static", "client"]: + resources.update({ + STATIC_PREFIX: self.get_resource_for_static_content(), + }) + + if name in ["media", "federation", "client"]: + resources.update({ + MEDIA_PREFIX: self.get_resource_for_media_repository(), + CONTENT_REPO_PREFIX: self.get_resource_for_content_repo(), + }) + + if name in ["keys", "federation"]: + resources.update({ + SERVER_KEY_PREFIX: self.get_resource_for_server_key(), + SERVER_KEY_V2_PREFIX: self.get_resource_for_server_key_v2(), + }) + + if name == "webclient": + resources[WEB_CLIENT_PREFIX] = self.get_resource_for_web_client() + + if name == "metrics" and metrics_resource: + resources[METRICS_PREFIX] = metrics_resource + + root_resource = create_resource_tree(resources) + if tls: reactor.listenSSL( - config.bind_port, + port, SynapseSite( "synapse.access.https", config, - self.root_resource, + root_resource, ), self.tls_context_factory, - interface=config.bind_host + interface=bind_address ) - logger.info("Synapse now listening on port %d", config.bind_port) - - if config.unsecure_port is not None: + else: reactor.listenTCP( - config.unsecure_port, + port, SynapseSite( - "synapse.access.http", + "synapse.access.https", config, - self.root_resource, + root_resource, ), - interface=config.bind_host + interface=bind_address ) - logger.info("Synapse now listening on port %d", config.unsecure_port) + logger.info("Synapse now listening on port %d", port) - metrics_resource = self.get_resource_for_metrics() - if metrics_resource and config.metrics_port is not None: - reactor.listenTCP( - config.metrics_port, - SynapseSite( - "synapse.access.metrics", - config, - metrics_resource, - ), - interface=config.metrics_bind_host, - ) - logger.info( - "Metrics now running on %s port %d", - config.metrics_bind_host, config.metrics_port, - ) + def start_listening(self): + config = self.get_config() + + for listener in config.listeners: + if listener["type"] == "http": + self._listener_http(config, listener) + elif listener["type"] == "manhole": + f = twisted.manhole.telnet.ShellFactory() + f.username = "matrix" + f.password = "rabbithole" + f.namespace['hs'] = self + reactor.listenTCP( + listener["port"], + f, + interface=listener.get("bind_address", '127.0.0.1') + ) + else: + logger.warn("Unrecognized listener type: %s", listener["type"]) def run_startup_checks(self, db_conn, database_engine): all_users_native = are_all_users_on_domain( @@ -322,11 +369,6 @@ def setup(config_options): events.USE_FROZEN_DICTS = config.use_frozen_dicts - if re.search(":[0-9]+$", config.server_name): - domain_with_port = config.server_name - else: - domain_with_port = "%s:%s" % (config.server_name, config.bind_port) - tls_context_factory = context_factory.ServerContextFactory(config) database_engine = create_engine(config.database_config["name"]) @@ -334,7 +376,6 @@ def setup(config_options): hs = SynapseHomeServer( config.server_name, - domain_with_port=domain_with_port, upload_dir=os.path.abspath("uploads"), db_config=config.database_config, tls_context_factory=tls_context_factory, @@ -344,29 +385,6 @@ def setup(config_options): database_engine=database_engine, ) - resources = { - CLIENT_PREFIX: hs.get_resource_for_client(), - CLIENT_V2_ALPHA_PREFIX: hs.get_resource_for_client_v2_alpha(), - FEDERATION_PREFIX: hs.get_resource_for_federation(), - CONTENT_REPO_PREFIX: hs.get_resource_for_content_repo(), - SERVER_KEY_PREFIX: hs.get_resource_for_server_key(), - SERVER_KEY_V2_PREFIX: hs.get_resource_for_server_key_v2(), - MEDIA_PREFIX: hs.get_resource_for_media_repository(), - STATIC_PREFIX: hs.get_resource_for_static_content(), - } - - if config.web_client: - resources[WEB_CLIENT_PREFIX] = hs.get_resource_for_web_client() - - metrics_resource = hs.get_resource_for_metrics() - if config.metrics_port is None and metrics_resource is not None: - resources[METRICS_PREFIX] = metrics_resource - - hs.root_resource = create_resource_tree( - resources, - redirect_root_to_web_client=True, - ) - logger.info("Preparing database: %r...", config.database_config) try: diff --git a/synapse/config/server.py b/synapse/config/server.py index 022ebcea9..26017c7ef 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -20,26 +20,73 @@ class ServerConfig(Config): def read_config(self, config): self.server_name = config["server_name"] - self.bind_port = config["bind_port"] - self.bind_host = config["bind_host"] - self.unsecure_port = config["unsecure_port"] self.manhole = config.get("manhole") self.pid_file = self.abspath(config.get("pid_file")) self.web_client = config["web_client"] self.soft_file_limit = config["soft_file_limit"] self.daemonize = config.get("daemonize") self.use_frozen_dicts = config.get("use_frozen_dicts", True) - self.gzip_responses = config["gzip_responses"] + + self.listeners = config.get("listeners", []) + + bind_port = config.get("bind_port") + if bind_port: + self.listeners = [] + bind_host = config.get("bind_host", "") + gzip_responses = config.get("gzip_responses", True) + + self.listeners.append({ + "port": bind_port, + "bind_address": bind_host, + "tls": True, + "type": "http", + "resources": [ + { + "names": ["client", "webclient"], + "compress": gzip_responses, + }, + { + "names": ["federation"], + "compress": False, + } + ] + }) + + unsecure_port = config.get("unsecure_port", bind_port - 400) + if unsecure_port: + self.listeners.append({ + "port": unsecure_port, + "bind_address": bind_host, + "tls": False, + "type": "http", + "resources": [ + { + "names": ["client", "webclient"], + "compress": gzip_responses, + }, + { + "names": ["federation"], + "compress": False, + } + ] + }) # Attempt to guess the content_addr for the v0 content repostitory content_addr = config.get("content_addr") if not content_addr: + for listener in self.listeners: + if listener["type"] == "http" and not listener.get("tls", False): + unsecure_port = listener["port"] + break + else: + raise RuntimeError("Could not determine 'content_addr'") + host = self.server_name if ':' not in host: - host = "%s:%d" % (host, self.unsecure_port) + host = "%s:%d" % (host, unsecure_port) else: host = host.split(':')[0] - host = "%s:%d" % (host, self.unsecure_port) + host = "%s:%d" % (host, unsecure_port) content_addr = "http://%s" % (host,) self.content_addr = content_addr @@ -61,9 +108,17 @@ class ServerConfig(Config): # e.g. matrix.org, localhost:8080, etc. server_name: "%(server_name)s" + # The port to listen for HTTPS requests on. + # For when matrix traffic is sent directly to synapse. + # bind_port: %(bind_port)s + + # The port to listen for HTTP requests on. + # For when matrix traffic passes through loadbalancer that unwraps TLS. + # unsecure_port: %(unsecure_port)s + # Local interface to listen on. # The empty string will cause synapse to listen on all interfaces. - bind_host: "" + # bind_host: "" # When running as a daemon, the file to store the pid in pid_file: %(pid_file)s @@ -83,31 +138,30 @@ class ServerConfig(Config): # Should synapse compress HTTP responses to clients that support it? # This should be disabled if running synapse behind a load balancer # that can do automatic compression. - gzip_responses: True + # gzip_responses: True listeners: - # For when matrix traffic is sent directly to synapse. - secure: - # The type of - type: http_resource + - port: %(unsecure_port)s + tls: false + bind_address: '' + type: http - # The port to listen for HTTPS requests on. - port: %(bind_port)s - - # Is this a TLS socket? - tls: true - - # Local interface to listen on. - # The empty string will cause synapse to listen on all interfaces. - bind_address: "" - - # For when matrix traffic passes through loadbalancer that unwraps TLS. - unsecure: - port: %(unsecure_port)s - tls: false - bind_address: "" + resources: + - names: [client, webclient] + compress: true + - names: [federation] + compress: false + - port: %(bind_port)s + tls: true + bind_address: '' + type: http + resources: + - names: [client, webclient] + compress: true + - names: [federation] + compress: false """ % locals() def read_arguments(self, args): From 9c5fc81c2ddd29eac62d368e7f8d24972f8894a6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 17:13:23 +0100 Subject: [PATCH 032/180] Correctly handle x_forwaded listener option --- synapse/app/homeserver.py | 31 +++++++++++++++++++++++++++---- synapse/config/server.py | 2 ++ synapse/server.py | 12 ++---------- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 4228bac67..12da0bc4b 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -34,7 +34,7 @@ from twisted.application import service from twisted.enterprise import adbapi from twisted.web.resource import Resource, EncodingResourceWrapper from twisted.web.static import File -from twisted.web.server import Site, GzipEncoderFactory +from twisted.web.server import Site, GzipEncoderFactory, Request from twisted.web.http import proxiedLogFormatter, combinedLogFormatter from synapse.http.server import JsonResource, RootRedirect from synapse.rest.media.v0.content_repository import ContentRepoResource @@ -199,7 +199,7 @@ class SynapseHomeServer(HomeServer): port, SynapseSite( "synapse.access.https", - config, + listener_config, root_resource, ), self.tls_context_factory, @@ -210,7 +210,7 @@ class SynapseHomeServer(HomeServer): port, SynapseSite( "synapse.access.https", - config, + listener_config, root_resource, ), interface=bind_address @@ -441,6 +441,28 @@ class SynapseService(service.Service): return self._port.stopListening() +class XForwardedForRequest(Request): + def __init__(self, *args, **kw): + Request.__init__(self, *args, **kw) + + """ + Add a layer on top of another request that only uses the value of an + X-Forwarded-For header as the result of C{getClientIP}. + """ + def getClientIP(self): + """ + @return: The client address (the first address) in the value of the + I{X-Forwarded-For header}. If the header is not present, return + C{b"-"}. + """ + return self.requestHeaders.getRawHeaders( + b"x-forwarded-for", [b"-"])[0].split(b",")[0].strip() + + +def XForwardedFactory(*args, **kwargs): + return XForwardedForRequest(*args, **kwargs) + + class SynapseSite(Site): """ Subclass of a twisted http Site that does access logging with python's @@ -448,7 +470,8 @@ class SynapseSite(Site): """ def __init__(self, logger_name, config, resource, *args, **kwargs): Site.__init__(self, resource, *args, **kwargs) - if config.captcha_ip_origin_is_x_forwarded: + if config.get("x_forwarded", False): + self.requestFactory = XForwardedFactory self._log_formatter = proxiedLogFormatter else: self._log_formatter = combinedLogFormatter diff --git a/synapse/config/server.py b/synapse/config/server.py index 26017c7ef..9dab167b2 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -157,6 +157,8 @@ class ServerConfig(Config): bind_address: '' type: http + x_forwarded: False + resources: - names: [client, webclient] compress: true diff --git a/synapse/server.py b/synapse/server.py index 8b3dc675c..4d1fb1cbf 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -132,16 +132,8 @@ class BaseHomeServer(object): setattr(BaseHomeServer, "get_%s" % (depname), _get) def get_ip_from_request(self, request): - # May be an X-Forwarding-For header depending on config - ip_addr = request.getClientIP() - if self.config.captcha_ip_origin_is_x_forwarded: - # use the header - if request.requestHeaders.hasHeader("X-Forwarded-For"): - ip_addr = request.requestHeaders.getRawHeaders( - "X-Forwarded-For" - )[0] - - return ip_addr + # X-Forwarded-For is handled by our custom request type. + return request.getClientIP() def is_mine(self, domain_specific_string): return domain_specific_string.domain == self.hostname From 942e39e87c735a6ead5375681ceea035a945fd7d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 17:13:54 +0100 Subject: [PATCH 033/180] PEP8 --- synapse/app/homeserver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 12da0bc4b..91cc06a49 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -63,7 +63,6 @@ import synapse import logging import os -import re import resource import subprocess @@ -532,8 +531,9 @@ def create_resource_tree(desired_tree, redirect_root_to_web_client=True): # to be replaced with the desired resource. existing_dummy_resource = resource_mappings[res_id] for child_name in existing_dummy_resource.listNames(): - child_res_id = _resource_id(existing_dummy_resource, - child_name) + child_res_id = _resource_id( + existing_dummy_resource, child_name + ) child_resource = resource_mappings[child_res_id] # steal the children res.putChild(child_name, child_resource) From 261ccd7f5f3dddc1c4538c2697ba00918120ddc8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 17:17:29 +0100 Subject: [PATCH 034/180] Fix tests --- tests/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/utils.py b/tests/utils.py index 3b5c33591..eb035cf48 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -114,6 +114,8 @@ class MockHttpResource(HttpServer): mock_request.method = http_method mock_request.uri = path + mock_request.getClientIP.return_value = "-" + mock_request.requestHeaders.getRawHeaders.return_value=[ "X-Matrix origin=test,key=,sig=" ] From a005b7269a9c1df517f6a01e72c1eea5f4cb0354 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Jun 2015 17:41:36 +0100 Subject: [PATCH 035/180] Add backwards compat support for metrics, manhole and webclient config options --- synapse/app/homeserver.py | 7 ------- synapse/config/metrics.py | 6 ------ synapse/config/server.py | 30 +++++++++++++++++++++++++++--- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 91cc06a49..95e9122d3 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -408,13 +408,6 @@ def setup(config_options): logger.info("Database prepared in %r.", config.database_config) - if config.manhole: - f = twisted.manhole.telnet.ShellFactory() - f.username = "matrix" - f.password = "rabbithole" - f.namespace['hs'] = hs - reactor.listenTCP(config.manhole, f, interface='127.0.0.1') - hs.start_listening() hs.get_pusherpool().start() diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index 0cfb30ce7..ae5a69152 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -28,10 +28,4 @@ class MetricsConfig(Config): # Enable collection and rendering of performance metrics enable_metrics: False - - # Separate port to accept metrics requests on - # metrics_port: 8081 - - # Which host to bind the metric listener to - # metrics_bind_host: 127.0.0.1 """ diff --git a/synapse/config/server.py b/synapse/config/server.py index 9dab167b2..95bc967d0 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -20,7 +20,6 @@ class ServerConfig(Config): def read_config(self, config): self.server_name = config["server_name"] - self.manhole = config.get("manhole") self.pid_file = self.abspath(config.get("pid_file")) self.web_client = config["web_client"] self.soft_file_limit = config["soft_file_limit"] @@ -35,6 +34,8 @@ class ServerConfig(Config): bind_host = config.get("bind_host", "") gzip_responses = config.get("gzip_responses", True) + names = ["client", "webclient"] if self.web_client else ["client"] + self.listeners.append({ "port": bind_port, "bind_address": bind_host, @@ -42,7 +43,7 @@ class ServerConfig(Config): "type": "http", "resources": [ { - "names": ["client", "webclient"], + "names": names, "compress": gzip_responses, }, { @@ -61,7 +62,7 @@ class ServerConfig(Config): "type": "http", "resources": [ { - "names": ["client", "webclient"], + "names": names, "compress": gzip_responses, }, { @@ -71,6 +72,29 @@ class ServerConfig(Config): ] }) + manhole = config.get("manhole") + if manhole: + self.listeners.append({ + "port": manhole, + "bind_address": "127.0.0.1", + "type": "manhole", + }) + + metrics_port = config.get("metrics_port") + if metrics_port: + self.listeners.append({ + "port": metrics_port, + "bind_address": config.get("metrics_bind_host", "127.0.0.1"), + "tls": False, + "type": "http", + "resources": [ + { + "names": ["metrics"], + "compress": False, + }, + ] + }) + # Attempt to guess the content_addr for the v0 content repostitory content_addr = config.get("content_addr") if not content_addr: From 4df76b0a5d7f62622d0efe7456ef38d2fff06dc9 Mon Sep 17 00:00:00 2001 From: Ivan Shapovalov Date: Sat, 13 Jun 2015 11:08:49 +0300 Subject: [PATCH 036/180] MANIFEST.in: include python schema delta scripts (we now have one in 20/) --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 8243a942e..a9b543af8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ include *.rst include demo/README recursive-include synapse/storage/schema *.sql +recursive-include synapse/storage/schema *.py recursive-include demo *.dh recursive-include demo *.py From 2eda996a6390dd746a4274db5b276a4388e90d70 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 13 Jun 2015 12:21:58 +0100 Subject: [PATCH 037/180] Add a dummy.sql into delta/20 as pip isn't packinging the pushers.py --- synapse/storage/schema/delta/20/dummy.sql | 1 + 1 file changed, 1 insertion(+) create mode 100644 synapse/storage/schema/delta/20/dummy.sql diff --git a/synapse/storage/schema/delta/20/dummy.sql b/synapse/storage/schema/delta/20/dummy.sql new file mode 100644 index 000000000..e0ac49d1e --- /dev/null +++ b/synapse/storage/schema/delta/20/dummy.sql @@ -0,0 +1 @@ +SELECT 1; From 6c01ceb8d0aefd68d3e9bc63a0f0b84edfd4e808 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 13 Jun 2015 12:22:14 +0100 Subject: [PATCH 038/180] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index e6088dc6c..b6ac0dab3 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.9.2" +__version__ = "0.9.2-r1" From 186f61a3ac0810d991584c479ca4508c91252b87 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jun 2015 11:25:53 +0100 Subject: [PATCH 039/180] Document listener config. Remove deprecated config options --- synapse/config/captcha.py | 8 ---- synapse/config/server.py | 82 +++++++++++++++++++++++---------------- 2 files changed, 48 insertions(+), 42 deletions(-) diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index ba221121c..cf72dc434 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -21,10 +21,6 @@ class CaptchaConfig(Config): self.recaptcha_private_key = config["recaptcha_private_key"] self.recaptcha_public_key = config["recaptcha_public_key"] self.enable_registration_captcha = config["enable_registration_captcha"] - # XXX: This is used for more than just captcha - self.captcha_ip_origin_is_x_forwarded = ( - config["captcha_ip_origin_is_x_forwarded"] - ) self.captcha_bypass_secret = config.get("captcha_bypass_secret") self.recaptcha_siteverify_api = config["recaptcha_siteverify_api"] @@ -43,10 +39,6 @@ class CaptchaConfig(Config): # public/private key. enable_registration_captcha: False - # When checking captchas, use the X-Forwarded-For (XFF) header - # as the client IP and not the actual client IP. - captcha_ip_origin_is_x_forwarded: False - # A secret key used to bypass the captcha test entirely. #captcha_bypass_secret: "YOUR_SECRET_HERE" diff --git a/synapse/config/server.py b/synapse/config/server.py index 95bc967d0..b5af38737 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -132,18 +132,6 @@ class ServerConfig(Config): # e.g. matrix.org, localhost:8080, etc. server_name: "%(server_name)s" - # The port to listen for HTTPS requests on. - # For when matrix traffic is sent directly to synapse. - # bind_port: %(bind_port)s - - # The port to listen for HTTP requests on. - # For when matrix traffic passes through loadbalancer that unwraps TLS. - # unsecure_port: %(unsecure_port)s - - # Local interface to listen on. - # The empty string will cause synapse to listen on all interfaces. - # bind_host: "" - # When running as a daemon, the file to store the pid in pid_file: %(pid_file)s @@ -155,39 +143,65 @@ class ServerConfig(Config): # hard limit. soft_file_limit: 0 - # Turn on the twisted telnet manhole service on localhost on the given - # port. - #manhole: 9000 - - # Should synapse compress HTTP responses to clients that support it? - # This should be disabled if running synapse behind a load balancer - # that can do automatic compression. - # gzip_responses: True - + # List of ports that Synapse should listen on, their purpose and their + # configuration. listeners: - - port: %(unsecure_port)s + # Main HTTPS listener + # For when matrix traffic is sent directly to synapse. + - + # The port to listen for HTTPS requests on. + port: %(bind_port)s + + # Local interface to listen on. + # The empty string will cause synapse to listen on all interfaces. + bind_address: '' + + # This is a 'http' listener, allows us to specify 'resources'. + type: http + + tls: true + + # Use the X-Forwarded-For (XFF) header as the client IP and not the + # actual client IP. + x_forwarded: false + + # List of HTTP resources to serve on this listener. + resources: + - + # List of resources to host on this listener. + names: + - client # The client-server APIs, both v1 and v2 + - webclient # The bundled webclient. + + # Should synapse compress HTTP responses to clients that support it? + # This should be disabled if running synapse behind a load balancer + # that can do automatic compression. + compress: true + + - names: [federation] # Federation APIs + compress: false + + # Unsecure HTTP listener, + # For when matrix traffic passes through loadbalancer that unwraps TLS. + - + port: %(unsecure_port)s tls: false bind_address: '' type: http - resources: - - names: [client, webclient] - compress: true - - names: [federation] - compress: false - - - port: %(bind_port)s - tls: true - bind_address: '' - type: http - - x_forwarded: False + x_forwarded: false resources: - names: [client, webclient] compress: true - names: [federation] compress: false + + # Turn on the twisted telnet manhole service on localhost on the given + # port. + # - port: 9000 + # bind_address: 127.0.0.1 + # type: manhole """ % locals() def read_arguments(self, args): From 9d0326baa63f7e982d25f80abfc6fab09d3f25b6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jun 2015 11:27:29 +0100 Subject: [PATCH 040/180] Remove redundant newline --- synapse/config/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index b5af38737..f4d4a8710 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -183,8 +183,7 @@ class ServerConfig(Config): # Unsecure HTTP listener, # For when matrix traffic passes through loadbalancer that unwraps TLS. - - - port: %(unsecure_port)s + - port: %(unsecure_port)s tls: false bind_address: '' type: http From 83f119a84a592ccf0d2dc06e699bcf11eb4380b0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jun 2015 13:05:11 +0100 Subject: [PATCH 041/180] Log requests and responses sent via http.client --- synapse/http/client.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index e746f2416..9091ae2d3 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -61,21 +61,31 @@ class SimpleHttpClient(object): self.agent = Agent(reactor, pool=pool) self.version_string = hs.version_string - def request(self, method, *args, **kwargs): + def request(self, method, uri, *args, **kwargs): # A small wrapper around self.agent.request() so we can easily attach # counters to it outgoing_requests_counter.inc(method) d = preserve_context_over_fn( self.agent.request, - method, *args, **kwargs + method, uri, *args, **kwargs ) + logger.info("Sending request %s %s", method, uri) + def _cb(response): incoming_responses_counter.inc(method, response.code) + logger.info( + "Received response to %s %s: %s", + method, uri, response.code + ) return response def _eb(failure): incoming_responses_counter.inc(method, "ERR") + logger.info( + "Error sending request to %s %s: %s %s", + method, uri, failure.type, failure.getErrorMessage() + ) return failure d.addCallbacks(_cb, _eb) @@ -84,7 +94,9 @@ class SimpleHttpClient(object): @defer.inlineCallbacks def post_urlencoded_get_json(self, uri, args={}): + # TODO: Do we ever want to log message contents? logger.debug("post_urlencoded_get_json args: %s", args) + query_bytes = urllib.urlencode(args, True) response = yield self.request( @@ -105,7 +117,7 @@ class SimpleHttpClient(object): def post_json_get_json(self, uri, post_json): json_str = encode_canonical_json(post_json) - logger.info("HTTP POST %s -> %s", json_str, uri) + logger.debug("HTTP POST %s -> %s", json_str, uri) response = yield self.request( "POST", From f00f8346f143dc306e184b6d479294ab11a4ff55 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jun 2015 13:37:58 +0100 Subject: [PATCH 042/180] Make http.server request logging more verbose, but redact access_tokens --- synapse/http/server.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index ae8f3b397..e6e8a59f6 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -32,6 +32,7 @@ from twisted.web.util import redirectTo import collections import logging +import re import urllib logger = logging.getLogger(__name__) @@ -82,9 +83,18 @@ def request_handler(request_handler): code = None start = self.clock.time_msec() try: + request_uri = request.uri + + # Don't log access_tokens + request_uri = re.sub( + r'(\?.*access_token=)[^&]*(.*)$', + r'\1\2', + request_uri + ) + logger.info( - "Received request: %s %s", - request.method, request.path + "%s - Received request: %s %s", + request.getClientIP(), request.method, request_uri ) d = request_handler(self, request) with PreserveLoggingContext(): From 7f7ec84d6fcc707f225d5697f690c6888a7d1977 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jun 2015 14:16:29 +0100 Subject: [PATCH 043/180] Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index b6ac0dab3..d111335a1 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.9.2-r1" +__version__ = "0.9.2-r2" From 2acee97c2b304f3da07917df191f625b21b92335 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jun 2015 14:20:25 +0100 Subject: [PATCH 044/180] Changelog --- CHANGES.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index ac398728f..62aecda27 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +Changes in synapse v0.9.2-r2 (2015-06-15) +========================================= + +Fix packaging so that schema delta python files get included in the package. + Changes in synapse v0.9.2 (2015-06-12) ====================================== From b5209c57441d9e7bace28a03774d2605a6572514 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jun 2015 16:36:49 +0100 Subject: [PATCH 045/180] Create SynapseRequest that overrides __repr__ to not print access_token --- synapse/app/homeserver.py | 49 ++++++++++++++++++++++++++++++++++----- synapse/http/server.py | 14 +++-------- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 95e9122d3..7c1ad6bc1 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -63,6 +63,7 @@ import synapse import logging import os +import re import resource import subprocess @@ -433,9 +434,34 @@ class SynapseService(service.Service): return self._port.stopListening() -class XForwardedForRequest(Request): - def __init__(self, *args, **kw): +class SynapseRequest(Request): + def __init__(self, site_tag, *args, **kw): Request.__init__(self, *args, **kw) + self.site_tag = site_tag + self.authenticated_entity = None + + def __repr__(self): + # We overwrite this so that we don't log ``access_token`` + return '<%s at 0x%x method=%s uri=%s clientproto=%s site=%s>' % ( + self.__class__.__name__, + id(self), + self.method, + self.get_redacted_uri(), + self.clientproto, + self.site_tag, + ) + + def get_redacted_uri(self): + return re.sub( + r'(\?.*access_token=)[^&]*(.*)$', + r'\1\2', + self.uri + ) + + +class XForwardedForRequest(SynapseRequest): + def __init__(self, *args, **kw): + SynapseRequest.__init__(self, *args, **kw) """ Add a layer on top of another request that only uses the value of an @@ -451,8 +477,16 @@ class XForwardedForRequest(Request): b"x-forwarded-for", [b"-"])[0].split(b",")[0].strip() -def XForwardedFactory(*args, **kwargs): - return XForwardedForRequest(*args, **kwargs) +class SynapseRequestFactory(object): + def __init__(self, site_tag, x_forwarded_for): + self.site_tag = site_tag + self.x_forwarded_for = x_forwarded_for + + def __call__(self, *args, **kwargs): + if self.x_forwarded_for: + return XForwardedForRequest(self.site_tag, *args, **kwargs) + else: + return SynapseRequest(self.site_tag, *args, **kwargs) class SynapseSite(Site): @@ -462,8 +496,11 @@ class SynapseSite(Site): """ def __init__(self, logger_name, config, resource, *args, **kwargs): Site.__init__(self, resource, *args, **kwargs) - if config.get("x_forwarded", False): - self.requestFactory = XForwardedFactory + + proxied = config.get("x_forwarded", False) + self.requestFactory = SynapseRequestFactory(None, proxied) + + if proxied: self._log_formatter = proxiedLogFormatter else: self._log_formatter = combinedLogFormatter diff --git a/synapse/http/server.py b/synapse/http/server.py index e6e8a59f6..7f8b9dbb2 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -32,7 +32,6 @@ from twisted.web.util import redirectTo import collections import logging -import re import urllib logger = logging.getLogger(__name__) @@ -83,18 +82,11 @@ def request_handler(request_handler): code = None start = self.clock.time_msec() try: - request_uri = request.uri - - # Don't log access_tokens - request_uri = re.sub( - r'(\?.*access_token=)[^&]*(.*)$', - r'\1\2', - request_uri - ) - logger.info( "%s - Received request: %s %s", - request.getClientIP(), request.method, request_uri + request.getClientIP(), + request.method, + request.get_redacted_uri() ) d = request_handler(self, request) with PreserveLoggingContext(): From cee69441d3d3b4d966b6ec69c7dbf4eb3b876bb3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jun 2015 17:11:44 +0100 Subject: [PATCH 046/180] Log more when we have processed the request --- synapse/api/auth.py | 2 ++ synapse/app/homeserver.py | 10 ++++++++-- synapse/federation/transport/server.py | 1 + synapse/http/server.py | 14 ++++++++++++-- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d5bf0be85..4da62e5d8 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -370,6 +370,8 @@ class Auth(object): user_agent=user_agent ) + request.authenticated_entity = user.to_string() + defer.returnValue((user, ClientInfo(device_id, token_id))) except KeyError: raise AuthError( diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 7c1ad6bc1..fca6f06e3 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -143,6 +143,7 @@ class SynapseHomeServer(HomeServer): port = listener_config["port"] bind_address = listener_config.get("bind_address", "") tls = listener_config.get("tls", False) + site_tag = listener_config.get("tag", port) if tls and config.no_tls: return @@ -199,6 +200,7 @@ class SynapseHomeServer(HomeServer): port, SynapseSite( "synapse.access.https", + site_tag, listener_config, root_resource, ), @@ -210,6 +212,7 @@ class SynapseHomeServer(HomeServer): port, SynapseSite( "synapse.access.https", + site_tag, listener_config, root_resource, ), @@ -458,6 +461,9 @@ class SynapseRequest(Request): self.uri ) + def get_user_agent(self): + return self.requestHeaders.getRawHeaders("User-Agent", [None])[-1] + class XForwardedForRequest(SynapseRequest): def __init__(self, *args, **kw): @@ -494,11 +500,11 @@ class SynapseSite(Site): Subclass of a twisted http Site that does access logging with python's standard logging """ - def __init__(self, logger_name, config, resource, *args, **kwargs): + def __init__(self, logger_name, tag, config, resource, *args, **kwargs): Site.__init__(self, resource, *args, **kwargs) proxied = config.get("x_forwarded", False) - self.requestFactory = SynapseRequestFactory(None, proxied) + self.requestFactory = SynapseRequestFactory(tag, proxied) if proxied: self._log_formatter = proxiedLogFormatter diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 31190e700..bad93c6b2 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -94,6 +94,7 @@ class TransportLayerServer(object): yield self.keyring.verify_json_for_server(origin, json_request) logger.info("Request from %s", origin) + request.authenticated_entity = origin defer.returnValue((origin, content)) diff --git a/synapse/http/server.py b/synapse/http/server.py index 7f8b9dbb2..34645a371 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -125,8 +125,18 @@ def request_handler(request_handler): code = str(code) if code else "-" end = self.clock.time_msec() logger.info( - "Processed request: %dms %s %s %s", - end-start, code, request.method, request.path + "%s - %s - {%s}" + " Processed request: %dms %sB %s \"%s %s %s\" \"%s\"", + request.getClientIP(), + request.site_tag, + request.authenticated_entity, + end-start, + request.sentLength, + code, + request.method, + request.get_redacted_uri(), + request.clientproto, + request.get_user_agent(), ) return wrapped_request_handler From aaa749d366f768dd164f899c1d8e5eedd44ee5f5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jun 2015 18:18:05 +0100 Subject: [PATCH 047/180] Disable twisted access logging. Move access logging to SynapseRequest object --- synapse/app/homeserver.py | 64 ++++++++++++++++++++-------- synapse/http/server.py | 90 ++++++++++++++------------------------- 2 files changed, 79 insertions(+), 75 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fca6f06e3..7effedf7d 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -35,7 +35,6 @@ from twisted.enterprise import adbapi from twisted.web.resource import Resource, EncodingResourceWrapper from twisted.web.static import File from twisted.web.server import Site, GzipEncoderFactory, Request -from twisted.web.http import proxiedLogFormatter, combinedLogFormatter from synapse.http.server import JsonResource, RootRedirect from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v1.media_repository import MediaRepositoryResource @@ -61,11 +60,13 @@ import twisted.manhole.telnet import synapse +import contextlib import logging import os import re import resource import subprocess +import time logger = logging.getLogger("synapse.app.homeserver") @@ -438,10 +439,11 @@ class SynapseService(service.Service): class SynapseRequest(Request): - def __init__(self, site_tag, *args, **kw): + def __init__(self, site, *args, **kw): Request.__init__(self, *args, **kw) - self.site_tag = site_tag + self.site = site self.authenticated_entity = None + self.start_time = 0 def __repr__(self): # We overwrite this so that we don't log ``access_token`` @@ -451,7 +453,7 @@ class SynapseRequest(Request): self.method, self.get_redacted_uri(), self.clientproto, - self.site_tag, + self.site.site_tag, ) def get_redacted_uri(self): @@ -464,6 +466,38 @@ class SynapseRequest(Request): def get_user_agent(self): return self.requestHeaders.getRawHeaders("User-Agent", [None])[-1] + def started_processing(self): + self.site.access_logger.info( + "%s - %s - Received request: %s %s", + self.getClientIP(), + self.site.site_tag, + self.method, + self.get_redacted_uri() + ) + self.start_time = int(time.time() * 1000) + + def finished_processing(self): + self.site.access_logger.info( + "%s - %s - {%s}" + " Processed request: %dms %sB %s \"%s %s %s\" \"%s\"", + self.getClientIP(), + self.site.site_tag, + self.authenticated_entity, + int(time.time() * 1000) - self.start_time, + self.sentLength, + self.code, + self.method, + self.get_redacted_uri(), + self.clientproto, + self.get_user_agent(), + ) + + @contextlib.contextmanager + def processing(self): + self.started_processing() + yield + self.finished_processing() + class XForwardedForRequest(SynapseRequest): def __init__(self, *args, **kw): @@ -484,15 +518,15 @@ class XForwardedForRequest(SynapseRequest): class SynapseRequestFactory(object): - def __init__(self, site_tag, x_forwarded_for): - self.site_tag = site_tag + def __init__(self, site, x_forwarded_for): + self.site = site self.x_forwarded_for = x_forwarded_for def __call__(self, *args, **kwargs): if self.x_forwarded_for: - return XForwardedForRequest(self.site_tag, *args, **kwargs) + return XForwardedForRequest(self.site, *args, **kwargs) else: - return SynapseRequest(self.site_tag, *args, **kwargs) + return SynapseRequest(self.site, *args, **kwargs) class SynapseSite(Site): @@ -500,21 +534,17 @@ class SynapseSite(Site): Subclass of a twisted http Site that does access logging with python's standard logging """ - def __init__(self, logger_name, tag, config, resource, *args, **kwargs): + def __init__(self, logger_name, site_tag, config, resource, *args, **kwargs): Site.__init__(self, resource, *args, **kwargs) - proxied = config.get("x_forwarded", False) - self.requestFactory = SynapseRequestFactory(tag, proxied) + self.site_tag = site_tag - if proxied: - self._log_formatter = proxiedLogFormatter - else: - self._log_formatter = combinedLogFormatter + proxied = config.get("x_forwarded", False) + self.requestFactory = SynapseRequestFactory(self, proxied) self.access_logger = logging.getLogger(logger_name) def log(self, request): - line = self._log_formatter(self._logDateTime, request) - self.access_logger.info(line) + pass def create_resource_tree(desired_tree, redirect_root_to_web_client=True): diff --git a/synapse/http/server.py b/synapse/http/server.py index 34645a371..807ff95c6 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -79,65 +79,39 @@ def request_handler(request_handler): _next_request_id += 1 with LoggingContext(request_id) as request_context: request_context.request = request_id - code = None - start = self.clock.time_msec() - try: - logger.info( - "%s - Received request: %s %s", - request.getClientIP(), - request.method, - request.get_redacted_uri() - ) - d = request_handler(self, request) - with PreserveLoggingContext(): - yield d - code = request.code - except CodeMessageException as e: - code = e.code - if isinstance(e, SynapseError): - logger.info( - "%s SynapseError: %s - %s", request, code, e.msg + with request.processing(): + try: + d = request_handler(self, request) + with PreserveLoggingContext(): + yield d + except CodeMessageException as e: + code = e.code + if isinstance(e, SynapseError): + logger.info( + "%s SynapseError: %s - %s", request, code, e.msg + ) + else: + logger.exception(e) + outgoing_responses_counter.inc(request.method, str(code)) + respond_with_json( + request, code, cs_exception(e), send_cors=True, + pretty_print=_request_user_agent_is_curl(request), + version_string=self.version_string, + ) + except: + logger.exception( + "Failed handle request %s.%s on %r: %r", + request_handler.__module__, + request_handler.__name__, + self, + request + ) + respond_with_json( + request, + 500, + {"error": "Internal server error"}, + send_cors=True ) - else: - logger.exception(e) - outgoing_responses_counter.inc(request.method, str(code)) - respond_with_json( - request, code, cs_exception(e), send_cors=True, - pretty_print=_request_user_agent_is_curl(request), - version_string=self.version_string, - ) - except: - code = 500 - logger.exception( - "Failed handle request %s.%s on %r: %r", - request_handler.__module__, - request_handler.__name__, - self, - request - ) - respond_with_json( - request, - 500, - {"error": "Internal server error"}, - send_cors=True - ) - finally: - code = str(code) if code else "-" - end = self.clock.time_msec() - logger.info( - "%s - %s - {%s}" - " Processed request: %dms %sB %s \"%s %s %s\" \"%s\"", - request.getClientIP(), - request.site_tag, - request.authenticated_entity, - end-start, - request.sentLength, - code, - request.method, - request.get_redacted_uri(), - request.clientproto, - request.get_user_agent(), - ) return wrapped_request_handler From fb7def33446ed8cb01bc6e57cd56a7737b3be8b6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Jun 2015 10:09:43 +0100 Subject: [PATCH 048/180] Remove access_token from synapse.rest.client.v1.transactions {get,store}_response logging --- synapse/rest/client/v1/transactions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/rest/client/v1/transactions.py b/synapse/rest/client/v1/transactions.py index d933fea18..b861069b8 100644 --- a/synapse/rest/client/v1/transactions.py +++ b/synapse/rest/client/v1/transactions.py @@ -39,10 +39,10 @@ class HttpTransactionStore(object): A tuple of (HTTP response code, response content) or None. """ try: - logger.debug("get_response Key: %s TxnId: %s", key, txn_id) + logger.debug("get_response TxnId: %s", txn_id) (last_txn_id, response) = self.transactions[key] if txn_id == last_txn_id: - logger.info("get_response: Returning a response for %s", key) + logger.info("get_response: Returning a response for %s", txn_id) return response except KeyError: pass @@ -58,7 +58,7 @@ class HttpTransactionStore(object): txn_id (str): The transaction ID for this request. response (tuple): A tuple of (HTTP response code, response content) """ - logger.debug("store_response Key: %s TxnId: %s", key, txn_id) + logger.debug("store_response TxnId: %s", txn_id) self.transactions[key] = (txn_id, response) def store_client_transaction(self, request, txn_id, response): From 9a3cd1c00db95e894882b7ab4308022a3b014974 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 16 Jun 2015 16:03:35 +0100 Subject: [PATCH 049/180] Correct -H SERVER_NAME in config-missing complaint message --- synapse/config/_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index d4163d627..d483c67c6 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -148,7 +148,7 @@ class Config(object): if not config_args.config_path: config_parser.error( "Must supply a config file.\nA config file can be automatically" - " generated using \"--generate-config -h SERVER_NAME" + " generated using \"--generate-config -H SERVER_NAME" " -c CONFIG-FILE\"" ) @@ -209,7 +209,7 @@ class Config(object): if not config_args.config_path: config_parser.error( "Must supply a config file.\nA config file can be automatically" - " generated using \"--generate-config -h SERVER_NAME" + " generated using \"--generate-config -H SERVER_NAME" " -c CONFIG-FILE\"" ) From 04604062984a6dc27174fa42e4c9f91fbe7b0c42 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Jun 2015 16:59:38 +0100 Subject: [PATCH 050/180] Don't do unecessary db ops in presence.get_state --- synapse/handlers/message.py | 1 + synapse/handlers/presence.py | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 867fdbefb..1ed9e961f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -403,6 +403,7 @@ class MessageHandler(BaseHandler): target_user=UserID.from_string(m.user_id), auth_user=auth_user, as_event=True, + check_auth=False, ) presence.append(member_presence) except SynapseError: diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 023ad33ab..7c0319831 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -191,24 +191,24 @@ class PresenceHandler(BaseHandler): defer.returnValue(False) @defer.inlineCallbacks - def get_state(self, target_user, auth_user, as_event=False): + def get_state(self, target_user, auth_user, as_event=False, check_auth=True): if self.hs.is_mine(target_user): - visible = yield self.is_presence_visible( - observer_user=auth_user, - observed_user=target_user - ) + if check_auth: + visible = yield self.is_presence_visible( + observer_user=auth_user, + observed_user=target_user + ) - if not visible: - raise SynapseError(404, "Presence information not visible") - state = yield self.store.get_presence_state(target_user.localpart) - if "mtime" in state: - del state["mtime"] - state["presence"] = state.pop("state") + if not visible: + raise SynapseError(404, "Presence information not visible") if target_user in self._user_cachemap: - cached_state = self._user_cachemap[target_user].get_state() - if "last_active" in cached_state: - state["last_active"] = cached_state["last_active"] + state = self._user_cachemap[target_user].get_state() + else: + state = yield self.store.get_presence_state(target_user.localpart) + if "mtime" in state: + del state["mtime"] + state["presence"] = state.pop("state") else: # TODO(paul): Have remote server send us permissions set state = self._get_or_offline_usercache(target_user).get_state() From b849a64f8d467a7b1159b4e8c4db4f3d73696f78 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Jun 2015 17:03:24 +0100 Subject: [PATCH 051/180] Use DeferredList --- synapse/handlers/message.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 1ed9e961f..de9c6da9e 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -396,20 +396,20 @@ class MessageHandler(BaseHandler): ] presence_handler = self.hs.get_handlers().presence_handler - presence = [] - for m in room_members: - try: - member_presence = yield presence_handler.get_state( + presence_defs = yield defer.DeferredList( + [ + presence_handler.get_state( target_user=UserID.from_string(m.user_id), auth_user=auth_user, as_event=True, check_auth=False, ) - presence.append(member_presence) - except SynapseError: - logger.exception( - "Failed to get member presence of %r", m.user_id - ) + for m in room_members + ], + consumeErrors=True, + ) + + presence = [p for success, p in presence_defs if success] time_now = self.clock.time_msec() From eceb554a2f3e5bc3d1eb8b4f87c756dfd4c48340 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Jun 2015 17:12:27 +0100 Subject: [PATCH 052/180] Use another deferred list --- synapse/handlers/message.py | 45 ++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index de9c6da9e..e324662f1 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -380,15 +380,6 @@ class MessageHandler(BaseHandler): if limit is None: limit = 10 - messages, token = yield self.store.get_recent_events_for_room( - room_id, - limit=limit, - end_token=now_token.room_key, - ) - - start_token = now_token.copy_and_replace("room_key", token[0]) - end_token = now_token.copy_and_replace("room_key", token[1]) - room_members = [ m for m in current_state.values() if m.type == EventTypes.Member @@ -396,20 +387,38 @@ class MessageHandler(BaseHandler): ] presence_handler = self.hs.get_handlers().presence_handler - presence_defs = yield defer.DeferredList( + + @defer.inlineCallbacks + def get_presence(): + presence_defs = yield defer.DeferredList( + [ + presence_handler.get_state( + target_user=UserID.from_string(m.user_id), + auth_user=auth_user, + as_event=True, + check_auth=False, + ) + for m in room_members + ], + consumeErrors=True, + ) + + defer.returnValue([p for success, p in presence_defs if success]) + + presence, (messages, token) = yield defer.gatherResults( [ - presence_handler.get_state( - target_user=UserID.from_string(m.user_id), - auth_user=auth_user, - as_event=True, - check_auth=False, + get_presence(), + self.store.get_recent_events_for_room( + room_id, + limit=limit, + end_token=now_token.room_key, ) - for m in room_members ], consumeErrors=True, - ) + ).addErrback(unwrapFirstError) - presence = [p for success, p in presence_defs if success] + start_token = now_token.copy_and_replace("room_key", token[0]) + end_token = now_token.copy_and_replace("room_key", token[1]) time_now = self.clock.time_msec() From d88e20cdb988a44155ef087ef28cd37c982cfe5d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 17 Jun 2015 17:26:03 +0100 Subject: [PATCH 053/180] Fix bug where synapse was sending AS user queries incorrectly. Bug introduced in 92b20713d7c6346aeb20dc09963081e472752bb5 which reversed the comparison when checking if a user existed in the users table. Added UTs to prevent this happening again. --- synapse/handlers/appservice.py | 2 +- tests/handlers/test_appservice.py | 43 +++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8269482e4..1240e5164 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -177,7 +177,7 @@ class ApplicationServicesHandler(object): return user_info = yield self.store.get_user_by_id(user_id) - if not user_info: + if user_info: defer.returnValue(False) return diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 06cb1dd4c..9e95d1e53 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -57,6 +57,49 @@ class AppServiceHandlerTestCase(unittest.TestCase): interested_service, event ) + @defer.inlineCallbacks + def test_query_user_exists_unknown_user(self): + user_id = "@someone:anywhere" + services = [self._mkservice(is_interested=True)] + services[0].is_interested_in_user = Mock(return_value=True) + self.mock_store.get_app_services = Mock(return_value=services) + self.mock_store.get_user_by_id = Mock(return_value=None) + + event = Mock( + sender=user_id, + type="m.room.message", + room_id="!foo:bar" + ) + self.mock_as_api.push = Mock() + self.mock_as_api.query_user = Mock() + yield self.handler.notify_interested_services(event) + self.mock_as_api.query_user.assert_called_once_with( + services[0], user_id + ) + + @defer.inlineCallbacks + def test_query_user_exists_known_user(self): + user_id = "@someone:anywhere" + services = [self._mkservice(is_interested=True)] + services[0].is_interested_in_user = Mock(return_value=True) + self.mock_store.get_app_services = Mock(return_value=services) + self.mock_store.get_user_by_id = Mock(return_value={ + "name": user_id + }) + + event = Mock( + sender=user_id, + type="m.room.message", + room_id="!foo:bar" + ) + self.mock_as_api.push = Mock() + self.mock_as_api.query_user = Mock() + yield self.handler.notify_interested_services(event) + self.assertFalse( + self.mock_as_api.query_user.called, + "query_user called when it shouldn't have been." + ) + @defer.inlineCallbacks def test_query_room_alias_exists(self): room_alias_str = "#foo:bar" From 050ebccf309916c57c58b67776e66d98fffbff0f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 18 Jun 2015 11:36:26 +0100 Subject: [PATCH 054/180] Fix notifier leak --- synapse/notifier.py | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 078abfc56..27c034ed5 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -308,49 +308,48 @@ class Notifier(object): else: current_token = user_stream.current_token - listener = [_NotificationListener(deferred)] - - if timeout and not current_token.is_after(from_token): - user_stream.listeners.add(listener[0]) - + result = None if current_token.is_after(from_token): result = yield callback(from_token, current_token) - else: - result = None - - timer = [None] if result: - user_stream.listeners.discard(listener[0]) defer.returnValue(result) - return if timeout: + timer = [None] + listeners = [] timed_out = [False] + def notify_listeners(): + user_stream.listeners.difference_update(listeners) + for listener in listeners: + listener.notify(current_token) + del listeners[:] + def _timeout_listener(): timed_out[0] = True timer[0] = None - user_stream.listeners.discard(listener[0]) - listener[0].notify(current_token) + notify_listeners() # We create multiple notification listeners so we have to manage # canceling the timeout ourselves. timer[0] = self.clock.call_later(timeout/1000., _timeout_listener) while not result and not timed_out[0]: - new_token = yield deferred deferred = defer.Deferred() - listener[0] = _NotificationListener(deferred) - user_stream.listeners.add(listener[0]) + notify_listeners() + listeners.append(_NotificationListener(deferred)) + user_stream.listeners.update(listeners) + new_token = yield deferred + result = yield callback(current_token, new_token) current_token = new_token - if timer[0] is not None: - try: - self.clock.cancel_call_later(timer[0]) - except: - logger.exception("Failed to cancel notifer timer") + if timer[0] is not None: + try: + self.clock.cancel_call_later(timer[0]) + except: + logger.exception("Failed to cancel notifer timer") defer.returnValue(result) From 22049ea700173017cf2f8e88fb8848e06b82f9b3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 18 Jun 2015 15:49:05 +0100 Subject: [PATCH 055/180] Refactor the notifier.wait_for_events code to be clearer. Add _NotifierUserStream.new_listener that accpets a token to avoid races. --- synapse/notifier.py | 120 ++++++++++++++++++--------------------- synapse/util/__init__.py | 8 ++- synapse/util/async.py | 13 ++++- 3 files changed, 72 insertions(+), 69 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 27c034ed5..e44156102 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function -from synapse.util.async import run_on_reactor +from synapse.util.async import run_on_reactor, ObservableDeferred from synapse.types import StreamToken import synapse.metrics @@ -45,20 +45,16 @@ class _NotificationListener(object): The events stream handler will have yielded to the deferred, so to notify the handler it is sufficient to resolve the deferred. """ + __slots__ = ["deferred"] def __init__(self, deferred): - self.deferred = deferred + object.__setattr__(self, "deferred", deferred) - def notified(self): - return self.deferred.called + def __getattr__(self, name): + return getattr(self.deferred, name) - def notify(self, token): - """ Inform whoever is listening about the new events. - """ - try: - self.deferred.callback(token) - except defer.AlreadyCalledError: - pass + def __setattr__(self, name, value): + setattr(self.deferred, name, value) class _NotifierUserStream(object): @@ -75,11 +71,12 @@ class _NotifierUserStream(object): appservice=None): self.user = str(user) self.appservice = appservice - self.listeners = set() self.rooms = set(rooms) self.current_token = current_token self.last_notified_ms = time_now_ms + self.notify_deferred = ObservableDeferred(defer.Deferred()) + def notify(self, stream_key, stream_id, time_now_ms): """Notify any listeners for this user of a new event from an event source. @@ -91,12 +88,10 @@ class _NotifierUserStream(object): self.current_token = self.current_token.copy_and_advance( stream_key, stream_id ) - if self.listeners: - self.last_notified_ms = time_now_ms - listeners = self.listeners - self.listeners = set() - for listener in listeners: - listener.notify(self.current_token) + self.last_notified_ms = time_now_ms + noify_deferred = self.notify_deferred + self.notify_deferred = ObservableDeferred(defer.Deferred()) + noify_deferred.callback(self.current_token) def remove(self, notifier): """ Remove this listener from all the indexes in the Notifier @@ -114,6 +109,18 @@ class _NotifierUserStream(object): self.appservice, set() ).discard(self) + def count_listeners(self): + return len(self.noify_deferred.observers()) + + def new_listener(self, token): + """Returns a deferred that is resolved when there is a new token + greater than the given token. + """ + if self.current_token.is_after(token): + return _NotificationListener(defer.succeed(self.current_token)) + else: + return _NotificationListener(self.notify_deferred.observe()) + class Notifier(object): """ This class is responsible for notifying any listeners when there are @@ -158,7 +165,7 @@ class Notifier(object): for x in self.appservice_to_user_streams.values(): all_user_streams |= x - return sum(len(stream.listeners) for stream in all_user_streams) + return sum(stream.count_listeners() for stream in all_user_streams) metrics.register_callback("listeners", count_listeners) metrics.register_callback( @@ -286,10 +293,6 @@ class Notifier(object): """Wait until the callback returns a non empty response or the timeout fires. """ - - deferred = defer.Deferred() - time_now_ms = self.clock.time_msec() - user = str(user) user_stream = self.user_to_user_stream.get(user) if user_stream is None: @@ -302,54 +305,38 @@ class Notifier(object): rooms=rooms, appservice=appservice, current_token=current_token, - time_now_ms=time_now_ms, + time_now_ms=self.clock.time_msec(), ) self._register_with_keys(user_stream) - else: - current_token = user_stream.current_token result = None - if current_token.is_after(from_token): - result = yield callback(from_token, current_token) - - if result: - defer.returnValue(result) - if timeout: - timer = [None] - listeners = [] - timed_out = [False] + listener = None + timer = self.clock.call_later( + timeout/1000., lambda: listener.cancel() + ) - def notify_listeners(): - user_stream.listeners.difference_update(listeners) - for listener in listeners: - listener.notify(current_token) - del listeners[:] - - def _timeout_listener(): - timed_out[0] = True - timer[0] = None - notify_listeners() - - # We create multiple notification listeners so we have to manage - # canceling the timeout ourselves. - timer[0] = self.clock.call_later(timeout/1000., _timeout_listener) - - while not result and not timed_out[0]: - deferred = defer.Deferred() - notify_listeners() - listeners.append(_NotificationListener(deferred)) - user_stream.listeners.update(listeners) - new_token = yield deferred - - result = yield callback(current_token, new_token) - current_token = new_token - - if timer[0] is not None: + prev_token = from_token + while not result: try: - self.clock.cancel_call_later(timer[0]) - except: - logger.exception("Failed to cancel notifer timer") + # We need to start listening to the streams *before* doing + # the callback, as otherwise we may miss something. + current_token = user_stream.current_token + + result = yield callback(prev_token, current_token) + if result: + break + + prev_token = current_token + listener = user_stream.new_listener(prev_token) + yield listener.deferred + except defer.CancelledError: + break + + self.clock.cancel_call_later(timer, ignore_errs=True) + else: + current_token = user_stream.current_token + result = yield callback(from_token, current_token) defer.returnValue(result) @@ -367,6 +354,9 @@ class Notifier(object): @defer.inlineCallbacks def check_for_updates(before_token, after_token): + if not after_token.is_after(before_token): + defer.returnValue(None) + events = [] end_token = from_token for name, source in self.event_sources.sources.items(): @@ -401,7 +391,7 @@ class Notifier(object): expired_streams = [] expire_before_ts = time_now_ms - self.UNUSED_STREAM_EXPIRY_MS for stream in self.user_to_user_stream.values(): - if stream.listeners: + if stream.count_listeners(): continue if stream.last_notified_ms < expire_before_ts: expired_streams.append(stream) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 260714ccc..07ff25cef 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -91,8 +91,12 @@ class Clock(object): with PreserveLoggingContext(): return reactor.callLater(delay, wrapped_callback, *args, **kwargs) - def cancel_call_later(self, timer): - timer.cancel() + def cancel_call_later(self, timer, ignore_errs=False): + try: + timer.cancel() + except: + if not ignore_errs: + raise def time_bound_deferred(self, given_deferred, time_out): if given_deferred.called: diff --git a/synapse/util/async.py b/synapse/util/async.py index 1c2044e5b..6f567bcaa 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -45,7 +45,7 @@ class ObservableDeferred(object): def __init__(self, deferred, consumeErrors=False): object.__setattr__(self, "_deferred", deferred) object.__setattr__(self, "_result", None) - object.__setattr__(self, "_observers", []) + object.__setattr__(self, "_observers", set()) def callback(r): self._result = (True, r) @@ -74,12 +74,21 @@ class ObservableDeferred(object): def observe(self): if not self._result: d = defer.Deferred() - self._observers.append(d) + + def remove(r): + self._observers.discard(d) + return r + d.addBoth(remove) + + self._observers.add(d) return d else: success, res = self._result return defer.succeed(res) if success else defer.fail(res) + def observers(self): + return self._observers + def __getattr__(self, name): return getattr(self._deferred, name) From 1f24c2e5896c1c9e5c7fded779fdb0ddcf2a3801 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 18 Jun 2015 16:09:53 +0100 Subject: [PATCH 056/180] Don't bother proxying lookups on _NotificationListener to underlying deferred --- synapse/notifier.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index e44156102..053475a2f 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -48,13 +48,7 @@ class _NotificationListener(object): __slots__ = ["deferred"] def __init__(self, deferred): - object.__setattr__(self, "deferred", deferred) - - def __getattr__(self, name): - return getattr(self.deferred, name) - - def __setattr__(self, name, value): - setattr(self.deferred, name, value) + self.deferred = deferred class _NotifierUserStream(object): @@ -313,14 +307,12 @@ class Notifier(object): if timeout: listener = None timer = self.clock.call_later( - timeout/1000., lambda: listener.cancel() + timeout/1000., lambda: listener.deferred.cancel() ) prev_token = from_token while not result: try: - # We need to start listening to the streams *before* doing - # the callback, as otherwise we may miss something. current_token = user_stream.current_token result = yield callback(prev_token, current_token) From 73513ececce51427971d49f0d55bfa76dafc391e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 18 Jun 2015 16:15:10 +0100 Subject: [PATCH 057/180] Documentation --- synapse/notifier.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 053475a2f..5475ee36c 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -305,10 +305,13 @@ class Notifier(object): result = None if timeout: - listener = None - timer = self.clock.call_later( - timeout/1000., lambda: listener.deferred.cancel() - ) + listener = None # Will be set to a _NotificationListener that + # we'll be waiting on. Allows us to cancel it. + + def timed_out(): + if listener: + listener.deferred.cancel() + timer = self.clock.call_later(timeout/1000., timed_out) prev_token = from_token while not result: @@ -319,6 +322,10 @@ class Notifier(object): if result: break + # Now we wait for the _NotifierUserStream to be told there + # is a new token. + # We need to supply the token we supplied to callback so + # that we don't miss any current_token updates. prev_token = current_token listener = user_stream.new_listener(prev_token) yield listener.deferred From 6f6ebd216d6a84ee72f1becfb31bf40ba960e3e2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 18 Jun 2015 17:00:32 +0100 Subject: [PATCH 058/180] PEP8 --- synapse/notifier.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 5475ee36c..d6655f3f5 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -305,8 +305,9 @@ class Notifier(object): result = None if timeout: - listener = None # Will be set to a _NotificationListener that - # we'll be waiting on. Allows us to cancel it. + # Will be set to a _NotificationListener that we'll be waiting on. + # Allows us to cancel it. + listener = None def timed_out(): if listener: From 9e5a353663c111cfe34b727f75eed3616201ef72 Mon Sep 17 00:00:00 2001 From: Eric Myhre Date: Thu, 18 Jun 2015 23:38:20 -0500 Subject: [PATCH 059/180] Make upload dir a configurable path. Fixes SYN-425. Signed-off-by: Eric Myhre --- synapse/app/homeserver.py | 3 +-- synapse/config/repository.py | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 95e9122d3..9aec23d06 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -112,7 +112,7 @@ class SynapseHomeServer(HomeServer): def build_resource_for_content_repo(self): return ContentRepoResource( - self, self.upload_dir, self.auth, self.content_addr + self, self.config.uploads_path, self.auth, self.content_addr ) def build_resource_for_media_repository(self): @@ -375,7 +375,6 @@ def setup(config_options): hs = SynapseHomeServer( config.server_name, - upload_dir=os.path.abspath("uploads"), db_config=config.database_config, tls_context_factory=tls_context_factory, config=config, diff --git a/synapse/config/repository.py b/synapse/config/repository.py index adaf4e4bb..6891abd71 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -21,13 +21,18 @@ class ContentRepositoryConfig(Config): self.max_upload_size = self.parse_size(config["max_upload_size"]) self.max_image_pixels = self.parse_size(config["max_image_pixels"]) self.media_store_path = self.ensure_directory(config["media_store_path"]) + self.uploads_path = self.ensure_directory(config["uploads_path"]) def default_config(self, config_dir_path, server_name): media_store = self.default_path("media_store") + uploads_path = self.default_path("uploads") return """ # Directory where uploaded images and attachments are stored. media_store_path: "%(media_store)s" + # Directory where in-progress uploads are stored. + uploads_path: "%(uploads_path)s" + # The largest allowed upload size in bytes max_upload_size: "10M" From ad460a83151d3c3d7c4980e6312537871a3234ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 19 Jun 2015 09:23:26 +0100 Subject: [PATCH 060/180] Add Eric Myhre to AUTHORS --- AUTHORS.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 3a457cd9f..d7224ff5d 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -38,3 +38,7 @@ Brabo Ivan Shapovalov * contrib/systemd: a sample systemd unit file and a logger configuration + +Eric Myhre + * Fix bug where ``media_store_path`` config option was ignored by v0 content + repository API. From 9d112f444036eec00bfc44f8947f60fd48f9c7e1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 19 Jun 2015 10:13:03 +0100 Subject: [PATCH 061/180] Add IDs to outbound transactions --- synapse/http/matrixfederationclient.py | 124 ++++++++++++++----------- 1 file changed, 71 insertions(+), 53 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 7f3d8fc88..902b27841 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -35,11 +35,13 @@ from syutil.crypto.jsonsign import sign_json import simplejson as json import logging +import sys import urllib import urlparse logger = logging.getLogger(__name__) +outbound_logger = logging.getLogger("synapse.http.outbound") metrics = synapse.metrics.get_metrics_for(__name__) @@ -109,6 +111,8 @@ class MatrixFederationHttpClient(object): self.clock = hs.get_clock() self.version_string = hs.version_string + self._next_id = 1 + @defer.inlineCallbacks def _create_request(self, destination, method, path_bytes, body_callback, headers_dict={}, param_bytes=b"", @@ -123,8 +127,13 @@ class MatrixFederationHttpClient(object): ("", "", path_bytes, param_bytes, query_bytes, "",) ) - logger.info("Sending request to %s: %s %s", - destination, method, url_bytes) + txn_id = "%s-%s" % (method, self._next_id) + self._next_id = (self._next_id + 1) % (sys.maxint - 1) + + outbound_logger.info( + "{%s} [%s] Sending request: %s %s", + txn_id, destination, method, url_bytes + ) logger.debug( "Types: %s", @@ -141,63 +150,72 @@ class MatrixFederationHttpClient(object): endpoint = self._getEndpoint(reactor, destination) - while True: - producer = None - if body_callback: - producer = body_callback(method, url_bytes, headers_dict) + log_result = None + try: + while True: + producer = None + if body_callback: + producer = body_callback(method, url_bytes, headers_dict) - try: - request_deferred = preserve_context_over_fn( - self.agent.request, - destination, - endpoint, - method, - path_bytes, - param_bytes, - query_bytes, - Headers(headers_dict), - producer - ) - - response = yield self.clock.time_bound_deferred( - request_deferred, - time_out=timeout/1000. if timeout else 60, - ) - - logger.debug("Got response to %s", method) - break - except Exception as e: - if not retry_on_dns_fail and isinstance(e, DNSLookupError): - logger.warn( - "DNS Lookup failed to %s with %s", + try: + request_deferred = preserve_context_over_fn( + self.agent.request, destination, - e + endpoint, + method, + path_bytes, + param_bytes, + query_bytes, + Headers(headers_dict), + producer ) - raise - logger.warn( - "Sending request failed to %s: %s %s: %s - %s", - destination, - method, - url_bytes, - type(e).__name__, - _flatten_response_never_received(e), - ) + response = yield self.clock.time_bound_deferred( + request_deferred, + time_out=timeout/1000. if timeout else 60, + ) - if retries_left and not timeout: - yield sleep(2 ** (5 - retries_left)) - retries_left -= 1 - else: - raise + logger.debug("{%s} Got response to %s", txn_id, method) + log_result = "%d %s" % (response.code, response.phrase,) + break + except Exception as e: + if not retry_on_dns_fail and isinstance(e, DNSLookupError): + logger.warn( + "DNS Lookup failed to %s with %s", + destination, + e + ) + log_result = "DNS Lookup failed to %s with %s" % ( + destination, e + ) + raise - logger.info( - "Received response %d %s for %s: %s %s", - response.code, - response.phrase, - destination, - method, - url_bytes - ) + logger.warn( + "{%s} Sending request failed to %s: %s %s: %s - %s", + txn_id, + destination, + method, + url_bytes, + type(e).__name__, + _flatten_response_never_received(e), + ) + + log_result = "%s - %s" % ( + type(e).__name__, _flatten_response_never_received(e), + ) + + if retries_left and not timeout: + yield sleep(2 ** (5 - retries_left)) + retries_left -= 1 + else: + raise + finally: + outbound_logger.info( + "{%s} [%s] Result: %s", + txn_id, + destination, + log_result, + ) if 200 <= response.code < 300: pass From eb928c9f52697bef99b982a1aa5229a20e65c447 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 19 Jun 2015 10:16:48 +0100 Subject: [PATCH 062/180] Add site_tag to logger --- synapse/app/homeserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 7effedf7d..c02fc889c 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -200,7 +200,7 @@ class SynapseHomeServer(HomeServer): reactor.listenSSL( port, SynapseSite( - "synapse.access.https", + "synapse.access.https.%s" % (site_tag,), site_tag, listener_config, root_resource, @@ -212,7 +212,7 @@ class SynapseHomeServer(HomeServer): reactor.listenTCP( port, SynapseSite( - "synapse.access.https", + "synapse.access.http.%s" % (site_tag,), site_tag, listener_config, root_resource, From 18968efa0afacc72b91d626d1a6adc2d5476b130 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 19 Jun 2015 10:18:02 +0100 Subject: [PATCH 063/180] Remove stale debug lines --- synapse/http/matrixfederationclient.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 902b27841..1b9069273 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -135,15 +135,6 @@ class MatrixFederationHttpClient(object): txn_id, destination, method, url_bytes ) - logger.debug( - "Types: %s", - [ - type(destination), type(method), type(path_bytes), - type(param_bytes), - type(query_bytes) - ] - ) - # XXX: Would be much nicer to retry only at the transaction-layer # (once we have reliable transactions in place) retries_left = 5 @@ -175,7 +166,6 @@ class MatrixFederationHttpClient(object): time_out=timeout/1000. if timeout else 60, ) - logger.debug("{%s} Got response to %s", txn_id, method) log_result = "%d %s" % (response.code, response.phrase,) break except Exception as e: From 653533a3dac1790f218aa4978f775f8098656b11 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 19 Jun 2015 11:45:55 +0100 Subject: [PATCH 064/180] Fix log context when sending requests --- synapse/http/client.py | 10 ++--- synapse/http/matrixfederationclient.py | 58 +++++++++++++------------- synapse/util/logcontext.py | 52 ++++++++++++++--------- 3 files changed, 68 insertions(+), 52 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index 9091ae2d3..49737d55d 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -109,7 +109,7 @@ class SimpleHttpClient(object): bodyProducer=FileBodyProducer(StringIO(query_bytes)) ) - body = yield readBody(response) + body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body)) @@ -128,7 +128,7 @@ class SimpleHttpClient(object): bodyProducer=FileBodyProducer(StringIO(json_str)) ) - body = yield readBody(response) + body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body)) @@ -161,7 +161,7 @@ class SimpleHttpClient(object): }) ) - body = yield readBody(response) + body = yield preserve_context_over_fn(readBody, response) if 200 <= response.code < 300: defer.returnValue(json.loads(body)) @@ -204,7 +204,7 @@ class SimpleHttpClient(object): bodyProducer=FileBodyProducer(StringIO(json_str)) ) - body = yield readBody(response) + body = yield preserve_context_over_fn(readBody, response) if 200 <= response.code < 300: defer.returnValue(json.loads(body)) @@ -238,7 +238,7 @@ class CaptchaServerHttpClient(SimpleHttpClient): ) try: - body = yield readBody(response) + body = yield preserve_context_over_fn(readBody, response) defer.returnValue(body) except PartialDownloadError as e: # twisted dislikes google's response, no content length. diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 1b9069273..ed47e701e 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -127,7 +127,7 @@ class MatrixFederationHttpClient(object): ("", "", path_bytes, param_bytes, query_bytes, "",) ) - txn_id = "%s-%s" % (method, self._next_id) + txn_id = "%s-O-%s" % (method, self._next_id) self._next_id = (self._next_id + 1) % (sys.maxint - 1) outbound_logger.info( @@ -139,7 +139,9 @@ class MatrixFederationHttpClient(object): # (once we have reliable transactions in place) retries_left = 5 - endpoint = self._getEndpoint(reactor, destination) + endpoint = preserve_context_over_fn( + self._getEndpoint, reactor, destination + ) log_result = None try: @@ -149,21 +151,25 @@ class MatrixFederationHttpClient(object): producer = body_callback(method, url_bytes, headers_dict) try: - request_deferred = preserve_context_over_fn( - self.agent.request, - destination, - endpoint, - method, - path_bytes, - param_bytes, - query_bytes, - Headers(headers_dict), - producer - ) + def send_request(): + request_deferred = self.agent.request( + destination, + endpoint, + method, + path_bytes, + param_bytes, + query_bytes, + Headers(headers_dict), + producer + ) - response = yield self.clock.time_bound_deferred( - request_deferred, - time_out=timeout/1000. if timeout else 60, + return self.clock.time_bound_deferred( + request_deferred, + time_out=timeout/1000. if timeout else 60, + ) + + response = yield preserve_context_over_fn( + send_request, ) log_result = "%d %s" % (response.code, response.phrase,) @@ -212,7 +218,7 @@ class MatrixFederationHttpClient(object): else: # :'( # Update transactions table? - body = yield readBody(response) + body = yield preserve_context_over_fn(readBody, response) raise HttpResponseException( response.code, response.phrase, body ) @@ -292,10 +298,7 @@ class MatrixFederationHttpClient(object): "Content-Type not application/json" ) - logger.debug("Getting resp body") - body = yield readBody(response) - logger.debug("Got resp body") - + body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body)) @defer.inlineCallbacks @@ -338,9 +341,7 @@ class MatrixFederationHttpClient(object): "Content-Type not application/json" ) - logger.debug("Getting resp body") - body = yield readBody(response) - logger.debug("Got resp body") + body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body)) @@ -398,9 +399,7 @@ class MatrixFederationHttpClient(object): "Content-Type not application/json" ) - logger.debug("Getting resp body") - body = yield readBody(response) - logger.debug("Got resp body") + body = yield preserve_context_over_fn(readBody, response) defer.returnValue(json.loads(body)) @@ -443,7 +442,10 @@ class MatrixFederationHttpClient(object): headers = dict(response.headers.getAllRawHeaders()) try: - length = yield _readBodyToFile(response, output_stream, max_size) + length = yield preserve_context_over_fn( + _readBodyToFile, + response, output_stream, max_size + ) except: logger.exception("Failed to download body") raise diff --git a/synapse/util/logcontext.py b/synapse/util/logcontext.py index a92d518b4..7e6062c1b 100644 --- a/synapse/util/logcontext.py +++ b/synapse/util/logcontext.py @@ -140,6 +140,37 @@ class PreserveLoggingContext(object): ) +class _PreservingContextDeferred(defer.Deferred): + """A deferred that ensures that all callbacks and errbacks are called with + the given logging context. + """ + def __init__(self, context): + self._log_context = context + defer.Deferred.__init__(self) + + def addCallbacks(self, callback, errback=None, + callbackArgs=None, callbackKeywords=None, + errbackArgs=None, errbackKeywords=None): + callback = self._wrap_callback(callback) + errback = self._wrap_callback(errback) + return defer.Deferred.addCallbacks( + self, callback, + errback=errback, + callbackArgs=callbackArgs, + callbackKeywords=callbackKeywords, + errbackArgs=errbackArgs, + errbackKeywords=errbackKeywords, + ) + + def _wrap_callback(self, f): + def g(res, *args, **kwargs): + with PreserveLoggingContext(): + LoggingContext.thread_local.current_context = self._log_context + res = f(res, *args, **kwargs) + return res + return g + + def preserve_context_over_fn(fn, *args, **kwargs): """Takes a function and invokes it with the given arguments, but removes and restores the current logging context while doing so. @@ -160,24 +191,7 @@ def preserve_context_over_deferred(deferred): """Given a deferred wrap it such that any callbacks added later to it will be invoked with the current context. """ - d = defer.Deferred() - current_context = LoggingContext.current_context() - - def cb(res): - with PreserveLoggingContext(): - LoggingContext.thread_local.current_context = current_context - res = d.callback(res) - return res - - def eb(failure): - with PreserveLoggingContext(): - LoggingContext.thread_local.current_context = current_context - res = d.errback(failure) - return res - - if deferred.called: - return deferred - - deferred.addCallbacks(cb, eb) + d = _PreservingContextDeferred(current_context) + deferred.chainDeferred(d) return d From a68abc79fd90465aed6ead3eec1a5704c64a1682 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 19 Jun 2015 11:48:55 +0100 Subject: [PATCH 065/180] Add comment on cancellation of observers --- synapse/util/async.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/util/async.py b/synapse/util/async.py index 6f567bcaa..5a1d545c9 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -38,6 +38,9 @@ class ObservableDeferred(object): deferred. If consumeErrors is true errors will be captured from the origin deferred. + + Cancelling or otherwise resolving an observer will not affect the original + ObservableDeferred. """ __slots__ = ["_deferred", "_observers", "_result"] From 2f556e0c55c816b1344c88b964e6f86e397367d7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 19 Jun 2015 16:22:53 +0100 Subject: [PATCH 066/180] Fix typo --- synapse/notifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index d6655f3f5..bdd03dcbe 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -104,7 +104,7 @@ class _NotifierUserStream(object): ).discard(self) def count_listeners(self): - return len(self.noify_deferred.observers()) + return len(self.notify_deferred.observers()) def new_listener(self, token): """Returns a deferred that is resolved when there is a new token From 9c72011fd72099f02b7b9ec0f55b83f472bd11a7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 23 Jun 2015 10:12:19 +0100 Subject: [PATCH 067/180] Bumb version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index d111335a1..1f09df79c 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.9.2-r2" +__version__ = "0.9.3-rc1" From f043b14bc024e5b05cf4356b301f3ea75a4c9674 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 23 Jun 2015 10:22:42 +0100 Subject: [PATCH 068/180] Update change log --- CHANGES.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 62aecda27..caebde334 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,21 @@ +Changes in synapse v0.9.3 (2015-XX-XX) +====================================== + +General: + +* Fix a memory leak in the notifier. (SYN-412) +* Improve performance of room initial sync. (SYN-418) +* General improvements to logging. +* Remove ``access_token`` query params from ``INFO`` level logging. + +Configuration: + +* Add support for specifying and configuring multiple listeners. (SYN-389) + +Application services: + +* Fix bug where synapse failed to send user queries to application services. + Changes in synapse v0.9.2-r2 (2015-06-15) ========================================= From 6924852592e4e3737ccd050938abc266ae71bf18 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 23 Jun 2015 11:01:04 +0100 Subject: [PATCH 069/180] Batch SELECTs in _get_auth_chain_ids_txn --- synapse/storage/event_federation.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 1ba073884..ace745953 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -49,14 +49,22 @@ class EventFederationStore(SQLBaseStore): results = set() base_sql = ( - "SELECT auth_id FROM event_auth WHERE event_id = ?" + "SELECT auth_id FROM event_auth WHERE event_id IN (%s)" ) front = set(event_ids) while front: new_front = set() - for f in front: - txn.execute(base_sql, (f,)) + front_list = list(front) + chunks = [ + front_list[x:x+100] + for x in xrange(0, len(front), 100) + ] + for chunk in chunks: + txn.execute( + base_sql % (",".join(["?"] * len(chunk)),), + chunk + ) new_front.update([r[0] for r in txn.fetchall()]) new_front -= results From 6825eef9556b6f913252f0bb04676e39d9ddb808 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 23 Jun 2015 14:26:14 +0100 Subject: [PATCH 070/180] Oops: underride rule had an identifier with override in it. --- synapse/push/baserules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index f3d1cf5c5..1f015a7f2 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -164,7 +164,7 @@ def make_base_append_underride_rules(user): ] }, { - 'rule_id': 'global/override/.m.rule.contains_display_name', + 'rule_id': 'global/underride/.m.rule.contains_display_name', 'conditions': [ { 'kind': 'contains_display_name' From 5130d80d79fe1f95ce03b8f1cfd4fbf0a32f5ac8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 25 Jun 2015 17:18:19 +0100 Subject: [PATCH 071/180] Add bulk insert events API --- synapse/federation/federation_client.py | 6 + synapse/handlers/federation.py | 225 +++++++------ synapse/storage/event_federation.py | 108 +++--- synapse/storage/events.py | 417 ++++++++++++++---------- synapse/storage/roommember.py | 44 +-- synapse/storage/signatures.py | 28 +- synapse/storage/state.py | 36 +- synapse/storage/util/id_generators.py | 31 ++ 8 files changed, 521 insertions(+), 374 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index d3b46b24c..7ee3c66bf 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -327,6 +327,9 @@ class FederationClient(FederationBase): @defer.inlineCallbacks def make_join(self, destinations, room_id, user_id): for destination in destinations: + if destination == self.server_name: + continue + try: ret = yield self.transport_layer.make_join( destination, room_id, user_id @@ -353,6 +356,9 @@ class FederationClient(FederationBase): @defer.inlineCallbacks def send_join(self, destinations, pdu): for destination in destinations: + if destination == self.server_name: + continue + try: time_now = self._clock.time_msec() _, content = yield self.transport_layer.send_join( diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b5d882fd6..079f46dff 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -138,26 +138,29 @@ class FederationHandler(BaseHandler): if state and auth_chain is not None: # If we have any state or auth_chain given to us by the replication # layer, then we should handle them (if we haven't before.) + + event_infos = [] + for e in itertools.chain(auth_chain, state): if e.event_id in seen_ids: continue - e.internal_metadata.outlier = True - try: - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - yield self._handle_new_event( - origin, e, auth_events=auth - ) - seen_ids.add(e.event_id) - except: - logger.exception( - "Failed to handle state event %s", - e.event_id, - ) + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + event_infos.append({ + "event": e, + "auth_events": auth, + }) + seen_ids.add(e.event_id) + + yield self._handle_new_events( + origin, + event_infos, + outliers=True + ) try: _, event_stream_id, max_stream_id = yield self._handle_new_event( @@ -292,38 +295,29 @@ class FederationHandler(BaseHandler): ).addErrback(unwrapFirstError) auth_events.update({a.event_id: a for a in results}) - yield defer.gatherResults( - [ - self._handle_new_event( - dest, a, - auth_events={ - (auth_events[a_id].type, auth_events[a_id].state_key): - auth_events[a_id] - for a_id, _ in a.auth_events - }, - ) - for a in auth_events.values() - if a.event_id not in seen_events - ], - consumeErrors=True, - ).addErrback(unwrapFirstError) + ev_infos = [] + for a in auth_events.values(): + if a.event_id in seen_events: + continue + ev_infos.append({ + "event": a, + "auth_events": { + (auth_events[a_id].type, auth_events[a_id].state_key): + auth_events[a_id] + for a_id, _ in a.auth_events + } + }) - yield defer.gatherResults( - [ - self._handle_new_event( - dest, event_map[e_id], - state=events_to_state[e_id], - backfilled=True, - auth_events={ - (auth_events[a_id].type, auth_events[a_id].state_key): - auth_events[a_id] - for a_id, _ in event_map[e_id].auth_events - }, - ) - for e_id in events_to_state - ], - consumeErrors=True - ).addErrback(unwrapFirstError) + for e_id in events_to_state: + ev_infos.append({ + "event": event_map[e_id], + "state": events_to_state[e_id], + "auth_events": { + (auth_events[a_id].type, auth_events[a_id].state_key): + auth_events[a_id] + for a_id, _ in event_map[e_id].auth_events + } + }) events.sort(key=lambda e: e.depth) @@ -331,10 +325,14 @@ class FederationHandler(BaseHandler): if event in events_to_state: continue - yield self._handle_new_event( - dest, event, - backfilled=True, - ) + ev_infos.append({ + "event": event, + }) + + yield self._handle_new_events( + dest, ev_infos, + backfilled=True, + ) defer.returnValue(events) @@ -600,32 +598,22 @@ class FederationHandler(BaseHandler): # FIXME pass - yield self._handle_auth_events( - origin, [e for e in auth_chain if e.event_id != event.event_id] - ) - - @defer.inlineCallbacks - def handle_state(e): + ev_infos = [] + for e in itertools.chain(state, auth_chain): if e.event_id == event.event_id: - return + continue e.internal_metadata.outlier = True - try: - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { + auth_ids = [e_id for e_id, _ in e.auth_events] + ev_infos.append({ + "event": e, + "auth_events": { (e.type, e.state_key): e for e in auth_chain if e.event_id in auth_ids } - yield self._handle_new_event( - origin, e, auth_events=auth - ) - except: - logger.exception( - "Failed to handle state event %s", - e.event_id, - ) + }) - yield defer.DeferredList([handle_state(e) for e in state]) + yield self._handle_new_events(origin, ev_infos, outliers=True) auth_ids = [e_id for e_id, _ in event.auth_events] auth_events = { @@ -940,11 +928,54 @@ class FederationHandler(BaseHandler): def _handle_new_event(self, origin, event, state=None, backfilled=False, current_state=None, auth_events=None): - logger.debug( - "_handle_new_event: %s, sigs: %s", - event.event_id, event.signatures, + outlier = event.internal_metadata.is_outlier() + + context = yield self._prep_event( + origin, event, + state=state, + backfilled=backfilled, + current_state=current_state, + auth_events=auth_events, ) + event_stream_id, max_stream_id = yield self.store.persist_event( + event, + context=context, + backfilled=backfilled, + is_new_state=(not outlier and not backfilled), + current_state=current_state, + ) + + defer.returnValue((context, event_stream_id, max_stream_id)) + + @defer.inlineCallbacks + def _handle_new_events(self, origin, event_infos, backfilled=False, + outliers=False): + contexts = yield defer.gatherResults( + [ + self._prep_event( + origin, + ev_info["event"], + state=ev_info.get("state"), + backfilled=backfilled, + auth_events=ev_info.get("auth_events"), + ) + for ev_info in event_infos + ] + ) + + yield self.store.persist_events( + [ + (ev_info["event"], context) + for ev_info, context in itertools.izip(event_infos, contexts) + ], + backfilled=backfilled, + is_new_state=(not outliers and not backfilled), + ) + + @defer.inlineCallbacks + def _prep_event(self, origin, event, state=None, backfilled=False, + current_state=None, auth_events=None): outlier = event.internal_metadata.is_outlier() context = yield self.state_handler.compute_event_context( @@ -954,13 +985,6 @@ class FederationHandler(BaseHandler): if not auth_events: auth_events = context.current_state - logger.debug( - "_handle_new_event: %s, auth_events: %s", - event.event_id, auth_events, - ) - - is_new_state = not outlier - # This is a hack to fix some old rooms where the initial join event # didn't reference the create event in its auth events. if event.type == EventTypes.Member and not event.auth_events: @@ -984,26 +1008,7 @@ class FederationHandler(BaseHandler): context.rejected = RejectedReason.AUTH_ERROR - # FIXME: Don't store as rejected with AUTH_ERROR if we haven't - # seen all the auth events. - yield self.store.persist_event( - event, - context=context, - backfilled=backfilled, - is_new_state=False, - current_state=current_state, - ) - raise - - event_stream_id, max_stream_id = yield self.store.persist_event( - event, - context=context, - backfilled=backfilled, - is_new_state=(is_new_state and not backfilled), - current_state=current_state, - ) - - defer.returnValue((context, event_stream_id, max_stream_id)) + defer.returnValue(context) @defer.inlineCallbacks def on_query_auth(self, origin, event_id, remote_auth_chain, rejects, @@ -1066,14 +1071,24 @@ class FederationHandler(BaseHandler): @log_function def do_auth(self, origin, event, context, auth_events): # Check if we have all the auth events. - have_events = yield self.store.have_events( - [e_id for e_id, _ in event.auth_events] - ) - + current_state = set(e.event_id for e in auth_events.values()) event_auth_events = set(e_id for e_id, _ in event.auth_events) + + if event_auth_events - current_state: + have_events = yield self.store.have_events( + event_auth_events - current_state + ) + else: + have_events = {} + + have_events.update({ + e.event_id: "" + for e in auth_events.values() + }) + seen_events = set(have_events.keys()) - missing_auth = event_auth_events - seen_events + missing_auth = event_auth_events - seen_events - current_state if missing_auth: logger.info("Missing auth: %s", missing_auth) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index ace745953..09705c60b 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -282,8 +282,7 @@ class EventFederationStore(SQLBaseStore): }, ) - def _handle_prev_events(self, txn, outlier, event_id, prev_events, - room_id): + def _handle_mult_prev_events(self, txn, events): """ For the given event, update the event edges table and forward and backward extremities tables. @@ -293,68 +292,75 @@ class EventFederationStore(SQLBaseStore): table="event_edges", values=[ { - "event_id": event_id, + "event_id": ev.event_id, "prev_event_id": e_id, - "room_id": room_id, + "room_id": ev.room_id, "is_state": False, } - for e_id, _ in prev_events + for ev in events + for e_id, _ in ev.prev_events ], ) - # Update the extremities table if this is not an outlier. - if not outlier: - for e_id, _ in prev_events: - # TODO (erikj): This could be done as a bulk insert - self._simple_delete_txn( - txn, - table="event_forward_extremities", - keyvalues={ - "event_id": e_id, - "room_id": room_id, - } + events_by_room = {} + for ev in events: + events_by_room.setdefault(ev.room_id, []).append(ev) + + for room_id, room_events in events_by_room.items(): + prevs = [ + e_id for ev in room_events for e_id, _ in ev.prev_events + if not ev.internal_metadata.is_outlier() + ] + if prevs: + txn.execute( + "DELETE FROM event_forward_extremities" + " WHERE room_id = ?" + " AND event_id in (%s)" % ( + ",".join(["?"] * len(prevs)), + ), + [room_id] + prevs, ) - # We only insert as a forward extremity the new event if there are - # no other events that reference it as a prev event - query = ( - "SELECT 1 FROM event_edges WHERE prev_event_id = ?" - ) + query = ( + "INSERT INTO event_forward_extremities (event_id, room_id)" + " SELECT ?, ? WHERE NOT EXISTS (" + " SELECT 1 FROM event_edges WHERE prev_event_id = ?" + " )" + ) - txn.execute(query, (event_id,)) + txn.executemany( + query, + [(ev.event_id, ev.room_id, ev.event_id) for ev in events] + ) - if not txn.fetchone(): - query = ( - "INSERT INTO event_forward_extremities" - " (event_id, room_id)" - " VALUES (?, ?)" - ) + query = ( + "INSERT INTO event_backward_extremities (event_id, room_id)" + " SELECT ?, ? WHERE NOT EXISTS (" + " SELECT 1 FROM event_backward_extremities" + " WHERE event_id = ? AND room_id = ?" + " )" + " AND NOT EXISTS (" + " SELECT 1 FROM events WHERE event_id = ? AND room_id = ? " + " AND outlier = ?" + " )" + ) - txn.execute(query, (event_id, room_id)) + txn.executemany(query, [ + (e_id, ev.room_id, e_id, ev.room_id, e_id, ev.room_id, False) + for ev in events for e_id, _ in ev.prev_events + if not ev.internal_metadata.is_outlier() + ]) - query = ( - "INSERT INTO event_backward_extremities (event_id, room_id)" - " SELECT ?, ? WHERE NOT EXISTS (" - " SELECT 1 FROM event_backward_extremities" - " WHERE event_id = ? AND room_id = ?" - " )" - " AND NOT EXISTS (" - " SELECT 1 FROM events WHERE event_id = ? AND room_id = ? " - " AND outlier = ?" - " )" - ) - - txn.executemany(query, [ - (e_id, room_id, e_id, room_id, e_id, room_id, False) - for e_id, _ in prev_events - ]) - - query = ( - "DELETE FROM event_backward_extremities" - " WHERE event_id = ? AND room_id = ?" - ) - txn.execute(query, (event_id, room_id)) + query = ( + "DELETE FROM event_backward_extremities" + " WHERE event_id = ? AND room_id = ?" + ) + txn.executemany( + query, + [(ev.event_id, ev.room_id) for ev in events] + ) + for room_id in events_by_room: txn.call_after( self.get_latest_event_ids_in_room.invalidate, room_id ) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 2caf0aae8..d760acb87 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -23,9 +23,7 @@ from synapse.events.utils import prune_event from synapse.util.logcontext import preserve_context_over_deferred from synapse.util.logutils import log_function from synapse.api.constants import EventTypes -from synapse.crypto.event_signing import compute_event_reference_hash -from syutil.base64util import decode_base64 from syutil.jsonutil import encode_json from contextlib import contextmanager @@ -46,6 +44,48 @@ EVENT_QUEUE_TIMEOUT_S = 0.1 # Timeout when waiting for requests for events class EventsStore(SQLBaseStore): + @defer.inlineCallbacks + def persist_events(self, events_and_contexts, backfilled=False, + is_new_state=True): + if not events_and_contexts: + return + + if backfilled: + if not self.min_token_deferred.called: + yield self.min_token_deferred + start = self.min_token - 1 + self.min_token -= len(events_and_contexts) + 1 + stream_orderings = range(start, self.min_token, -1) + + @contextmanager + def stream_ordering_manager(): + yield stream_orderings + stream_ordering_manager = stream_ordering_manager() + else: + stream_ordering_manager = yield self._stream_id_gen.get_next_mult( + self, len(events_and_contexts) + ) + + with stream_ordering_manager as stream_orderings: + for (event, _), stream in zip(events_and_contexts, stream_orderings): + event.internal_metadata.stream_ordering = stream + + chunks = [ + events_and_contexts[x:x+100] + for x in xrange(0, len(events_and_contexts), 100) + ] + + for chunk in chunks: + # We can't easily parallelize these since different chunks + # might contain the same event. :( + yield self.runInteraction( + "persist_events", + self._persist_events_txn, + events_and_contexts=chunk, + backfilled=backfilled, + is_new_state=is_new_state, + ) + @defer.inlineCallbacks @log_function def persist_event(self, event, context, backfilled=False, @@ -67,13 +107,13 @@ class EventsStore(SQLBaseStore): try: with stream_ordering_manager as stream_ordering: + event.internal_metadata.stream_ordering = stream_ordering yield self.runInteraction( "persist_event", self._persist_event_txn, event=event, context=context, backfilled=backfilled, - stream_ordering=stream_ordering, is_new_state=is_new_state, current_state=current_state, ) @@ -116,12 +156,7 @@ class EventsStore(SQLBaseStore): @log_function def _persist_event_txn(self, txn, event, context, backfilled, - stream_ordering=None, is_new_state=True, - current_state=None): - - # Remove the any existing cache entries for the event_id - txn.call_after(self._invalidate_get_event_cache, event.event_id) - + is_new_state=True, current_state=None): # We purposefully do this first since if we include a `current_state` # key, we *want* to update the `current_state_events` table if current_state: @@ -149,37 +184,78 @@ class EventsStore(SQLBaseStore): } ) - outlier = event.internal_metadata.is_outlier() - - if not outlier: - self._update_min_depth_for_room_txn( - txn, - event.room_id, - event.depth - ) - - have_persisted = self._simple_select_one_txn( + return self._persist_events_txn( txn, - table="events", - keyvalues={"event_id": event.event_id}, - retcols=["event_id", "outlier"], - allow_none=True, + [(event, context)], + backfilled=backfilled, + is_new_state=is_new_state, ) - metadata_json = encode_json( - event.internal_metadata.get_dict(), - using_frozen_dicts=USE_FROZEN_DICTS - ).decode("UTF-8") + @log_function + def _persist_events_txn(self, txn, events_and_contexts, backfilled, + is_new_state=True): - # If we have already persisted this event, we don't need to do any - # more processing. - # The processing above must be done on every call to persist event, - # since they might not have happened on previous calls. For example, - # if we are persisting an event that we had persisted as an outlier, - # but is no longer one. - if have_persisted: - if not outlier and have_persisted["outlier"]: - self._store_state_groups_txn(txn, event, context) + # Remove the any existing cache entries for the event_ids + for event, _ in events_and_contexts: + txn.call_after(self._invalidate_get_event_cache, event.event_id) + + depth_updates = {} + for event, _ in events_and_contexts: + if event.internal_metadata.is_outlier(): + continue + depth_updates[event.room_id] = max( + event.depth, depth_updates.get(event.room_id, event.depth) + ) + + for room_id, depth in depth_updates.items(): + self._update_min_depth_for_room_txn(txn, room_id, depth) + + txn.execute( + "SELECT event_id, outlier FROM events WHERE event_id in (%s)" % ( + ",".join(["?"] * len(events_and_contexts)), + ), + [event.event_id for event, _ in events_and_contexts] + ) + have_persisted = { + event_id: outlier + for event_id, outlier in txn.fetchall() + } + + event_map = {} + to_remove = set() + for event, context in events_and_contexts: + # Handle the case of the list including the same event multiple + # times. The tricky thing here is when they differ by whether + # they are an outlier. + if event.event_id in event_map: + other = event_map[event.event_id] + + if not other.internal_metadata.is_outlier(): + to_remove.add(event) + continue + elif not event.internal_metadata.is_outlier(): + to_remove.add(event) + continue + else: + to_remove.add(other) + + event_map[event.event_id] = event + + if event.event_id not in have_persisted: + continue + + to_remove.add(event) + + outlier_persisted = have_persisted[event.event_id] + if not event.internal_metadata.is_outlier() and outlier_persisted: + self._store_state_groups_txn( + txn, event, context, + ) + + metadata_json = encode_json( + event.internal_metadata.get_dict(), + using_frozen_dicts=USE_FROZEN_DICTS + ).decode("UTF-8") sql = ( "UPDATE event_json SET internal_metadata = ?" @@ -198,94 +274,91 @@ class EventsStore(SQLBaseStore): sql, (False, event.event_id,) ) + + events_and_contexts = filter( + lambda ec: ec[0] not in to_remove, + events_and_contexts + ) + + if not events_and_contexts: return - if not outlier: - self._store_state_groups_txn(txn, event, context) + self._store_mult_state_groups_txn(txn, [ + (event, context) + for event, context in events_and_contexts + if not event.internal_metadata.is_outlier() + ]) - self._handle_prev_events( + self._handle_mult_prev_events( txn, - outlier=outlier, - event_id=event.event_id, - prev_events=event.prev_events, - room_id=event.room_id, + events=[event for event, _ in events_and_contexts], ) - if event.type == EventTypes.Member: - self._store_room_member_txn(txn, event) - elif event.type == EventTypes.Name: - self._store_room_name_txn(txn, event) - elif event.type == EventTypes.Topic: - self._store_room_topic_txn(txn, event) - elif event.type == EventTypes.Redaction: - self._store_redaction(txn, event) + for event, _ in events_and_contexts: + if event.type == EventTypes.Name: + self._store_room_name_txn(txn, event) + elif event.type == EventTypes.Topic: + self._store_room_topic_txn(txn, event) + elif event.type == EventTypes.Redaction: + self._store_redaction(txn, event) - event_dict = { - k: v - for k, v in event.get_dict().items() - if k not in [ - "redacted", - "redacted_because", + self._store_room_members_txn( + txn, + [ + event + for event, _ in events_and_contexts + if event.type == EventTypes.Member ] - } + ) - self._simple_insert_txn( + def event_dict(event): + return { + k: v + for k, v in event.get_dict().items() + if k not in [ + "redacted", + "redacted_because", + ] + } + + self._simple_insert_many_txn( txn, table="event_json", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "internal_metadata": metadata_json, - "json": encode_json( - event_dict, using_frozen_dicts=USE_FROZEN_DICTS - ).decode("UTF-8"), - }, + values=[ + { + "event_id": event.event_id, + "room_id": event.room_id, + "internal_metadata": encode_json( + event.internal_metadata.get_dict(), + using_frozen_dicts=USE_FROZEN_DICTS + ).decode("UTF-8"), + "json": encode_json( + event_dict(event), using_frozen_dicts=USE_FROZEN_DICTS + ).decode("UTF-8"), + } + for event, _ in events_and_contexts + ], ) - content = encode_json( - event.content, using_frozen_dicts=USE_FROZEN_DICTS - ).decode("UTF-8") - - vals = { - "topological_ordering": event.depth, - "event_id": event.event_id, - "type": event.type, - "room_id": event.room_id, - "content": content, - "processed": True, - "outlier": outlier, - "depth": event.depth, - } - - unrec = { - k: v - for k, v in event.get_dict().items() - if k not in vals.keys() and k not in [ - "redacted", - "redacted_because", - "signatures", - "hashes", - "prev_events", - ] - } - - vals["unrecognized_keys"] = encode_json( - unrec, using_frozen_dicts=USE_FROZEN_DICTS - ).decode("UTF-8") - - sql = ( - "INSERT INTO events" - " (stream_ordering, topological_ordering, event_id, type," - " room_id, content, processed, outlier, depth)" - " VALUES (?,?,?,?,?,?,?,?,?)" - ) - - txn.execute( - sql, - ( - stream_ordering, event.depth, event.event_id, event.type, - event.room_id, content, True, outlier, event.depth - ) + self._simple_insert_many_txn( + txn, + table="events", + values=[ + { + "stream_ordering": event.internal_metadata.stream_ordering, + "topological_ordering": event.depth, + "depth": event.depth, + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "processed": True, + "outlier": event.internal_metadata.is_outlier(), + "content": encode_json( + event.content, using_frozen_dicts=USE_FROZEN_DICTS + ).decode("UTF-8"), + } + for event, _ in events_and_contexts + ], ) if context.rejected: @@ -293,19 +366,19 @@ class EventsStore(SQLBaseStore): txn, event.event_id, context.rejected ) - for hash_alg, hash_base64 in event.hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_event_content_hash_txn( - txn, event.event_id, hash_alg, hash_bytes, - ) + # for hash_alg, hash_base64 in event.hashes.items(): + # hash_bytes = decode_base64(hash_base64) + # self._store_event_content_hash_txn( + # txn, event.event_id, hash_alg, hash_bytes, + # ) - for prev_event_id, prev_hashes in event.prev_events: - for alg, hash_base64 in prev_hashes.items(): - hash_bytes = decode_base64(hash_base64) - self._store_prev_event_hash_txn( - txn, event.event_id, prev_event_id, alg, - hash_bytes - ) + # for prev_event_id, prev_hashes in event.prev_events: + # for alg, hash_base64 in prev_hashes.items(): + # hash_bytes = decode_base64(hash_base64) + # self._store_prev_event_hash_txn( + # txn, event.event_id, prev_event_id, alg, + # hash_bytes + # ) self._simple_insert_many_txn( txn, @@ -316,16 +389,22 @@ class EventsStore(SQLBaseStore): "room_id": event.room_id, "auth_id": auth_id, } + for event, _ in events_and_contexts for auth_id, _ in event.auth_events ], ) - (ref_alg, ref_hash_bytes) = compute_event_reference_hash(event) - self._store_event_reference_hash_txn( - txn, event.event_id, ref_alg, ref_hash_bytes + self._store_event_reference_hashes_txn( + txn, [event for event, _ in events_and_contexts] ) - if event.is_state(): + state_events_and_contexts = filter( + lambda i: i[0].is_state(), + events_and_contexts, + ) + + state_values = [] + for event, context in state_events_and_contexts: vals = { "event_id": event.event_id, "room_id": event.room_id, @@ -337,51 +416,55 @@ class EventsStore(SQLBaseStore): if hasattr(event, "replaces_state"): vals["prev_state"] = event.replaces_state - self._simple_insert_txn( - txn, - "state_events", - vals, - ) + state_values.append(vals) - self._simple_insert_many_txn( - txn, - table="event_edges", - values=[ - { - "event_id": event.event_id, - "prev_event_id": e_id, - "room_id": event.room_id, - "is_state": True, - } - for e_id, h in event.prev_state - ], - ) + self._simple_insert_many_txn( + txn, + table="state_events", + values=state_values, + ) - if is_new_state and not context.rejected: - txn.call_after( - self.get_current_state_for_key.invalidate, - event.room_id, event.type, event.state_key - ) + self._simple_insert_many_txn( + txn, + table="event_edges", + values=[ + { + "event_id": event.event_id, + "prev_event_id": prev_id, + "room_id": event.room_id, + "is_state": True, + } + for event, _ in state_events_and_contexts + for prev_id, _ in event.prev_state + ], + ) - if (event.type == EventTypes.Name - or event.type == EventTypes.Aliases): + if is_new_state: + for event, _ in state_events_and_contexts: + if not context.rejected: txn.call_after( - self.get_room_name_and_aliases.invalidate, - event.room_id - ) + self.get_current_state_for_key.invalidate, + event.room_id, event.type, event.state_key + ) - self._simple_upsert_txn( - txn, - "current_state_events", - keyvalues={ - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - values={ - "event_id": event.event_id, - } - ) + if event.type in [EventTypes.Name, EventTypes.Aliases]: + txn.call_after( + self.get_room_name_and_aliases.invalidate, + event.room_id + ) + + self._simple_upsert_txn( + txn, + "current_state_events", + keyvalues={ + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + values={ + "event_id": event.event_id, + } + ) return diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index d36a6c18a..4db07f6fb 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -35,38 +35,28 @@ RoomsForUser = namedtuple( class RoomMemberStore(SQLBaseStore): - def _store_room_member_txn(self, txn, event): + def _store_room_members_txn(self, txn, events): """Store a room member in the database. """ - try: - target_user_id = event.state_key - except: - logger.exception( - "Failed to parse target_user_id=%s", target_user_id - ) - raise - - logger.debug( - "_store_room_member_txn: target_user_id=%s, membership=%s", - target_user_id, - event.membership, - ) - - self._simple_insert_txn( + self._simple_insert_many_txn( txn, - "room_memberships", - { - "event_id": event.event_id, - "user_id": target_user_id, - "sender": event.user_id, - "room_id": event.room_id, - "membership": event.membership, - } + table="room_memberships", + values=[ + { + "event_id": event.event_id, + "user_id": event.state_key, + "sender": event.user_id, + "room_id": event.room_id, + "membership": event.membership, + } + for event in events + ] ) - txn.call_after(self.get_rooms_for_user.invalidate, target_user_id) - txn.call_after(self.get_joined_hosts_for_room.invalidate, event.room_id) - txn.call_after(self.get_users_in_room.invalidate, event.room_id) + for event in events: + txn.call_after(self.get_rooms_for_user.invalidate, event.state_key) + txn.call_after(self.get_joined_hosts_for_room.invalidate, event.room_id) + txn.call_after(self.get_users_in_room.invalidate, event.room_id) def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index f05182863..4f15e534b 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -18,6 +18,7 @@ from twisted.internet import defer from _base import SQLBaseStore from syutil.base64util import encode_base64 +from synapse.crypto.event_signing import compute_event_reference_hash class SignatureStore(SQLBaseStore): @@ -101,23 +102,26 @@ class SignatureStore(SQLBaseStore): txn.execute(query, (event_id, )) return {k: v for k, v in txn.fetchall()} - def _store_event_reference_hash_txn(self, txn, event_id, algorithm, - hash_bytes): + def _store_event_reference_hashes_txn(self, txn, events): """Store a hash for a PDU Args: txn (cursor): - event_id (str): Id for the Event. - algorithm (str): Hashing algorithm. - hash_bytes (bytes): Hash function output bytes. + events (list): list of Events. """ - self._simple_insert_txn( + + vals = [] + for event in events: + ref_alg, ref_hash_bytes = compute_event_reference_hash(event) + vals.append({ + "event_id": event.event_id, + "algorithm": ref_alg, + "hash": buffer(ref_hash_bytes), + }) + + self._simple_insert_many_txn( txn, - "event_reference_hashes", - { - "event_id": event_id, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }, + table="event_reference_hashes", + values=vals, ) def _get_event_signatures_txn(self, txn, event_id): diff --git a/synapse/storage/state.py b/synapse/storage/state.py index f2b17f29e..61214e2b2 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -100,16 +100,23 @@ class StateStore(SQLBaseStore): ) def _store_state_groups_txn(self, txn, event, context): - if context.current_state is None: - return + return self._store_mult_state_groups_txn(txn, [(event, context)]) - state_events = dict(context.current_state) + def _store_mult_state_groups_txn(self, txn, events_and_contexts): + state_groups = {} + for event, context in events_and_contexts: + if context.current_state is None: + continue - if event.is_state(): - state_events[(event.type, event.state_key)] = event + if context.state_group is not None: + state_groups[event.event_id] = context.state_group + continue + + state_events = dict(context.current_state) + + if event.is_state(): + state_events[(event.type, event.state_key)] = event - state_group = context.state_group - if not state_group: state_group = self._state_groups_id_gen.get_next_txn(txn) self._simple_insert_txn( txn, @@ -135,14 +142,19 @@ class StateStore(SQLBaseStore): for state in state_events.values() ], ) + state_groups[event.event_id] = state_group - self._simple_insert_txn( + self._simple_insert_many_txn( txn, table="event_to_state_groups", - values={ - "state_group": state_group, - "event_id": event.event_id, - }, + values=[ + { + "state_group": state_groups[event.event_id], + "event_id": event.event_id, + } + for event, context in events_and_contexts + if context.current_state is not None + ], ) @defer.inlineCallbacks diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index 89d1643f1..83eab6309 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -107,6 +107,37 @@ class StreamIdGenerator(object): defer.returnValue(manager()) + @defer.inlineCallbacks + def get_next_mult(self, store, n): + """ + Usage: + with yield stream_id_gen.get_next(store, n) as stream_ids: + # ... persist events ... + """ + if not self._current_max: + yield store.runInteraction( + "_compute_current_max", + self._get_or_compute_current_max, + ) + + with self._lock: + next_ids = range(self._current_max + 1, self._current_max + n + 1) + self._current_max += n + + for next_id in next_ids: + self._unfinished_ids.append(next_id) + + @contextlib.contextmanager + def manager(): + try: + yield next_ids + finally: + with self._lock: + for next_id in next_ids: + self._unfinished_ids.remove(next_id) + + defer.returnValue(manager()) + @defer.inlineCallbacks def get_max_token(self, store): """Returns the maximum stream id such that all stream ids less than or From b5f55a1d85386834b18828b196e1859a4410d98a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 26 Jun 2015 09:52:24 +0100 Subject: [PATCH 072/180] Implement bulk verify_signed_json API --- synapse/crypto/keyring.py | 426 ++++++++++++++++-------- synapse/federation/federation_base.py | 121 ++++--- synapse/federation/federation_client.py | 57 +++- synapse/storage/keys.py | 50 +-- 4 files changed, 439 insertions(+), 215 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index aff69c5f8..873c9b40f 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -25,11 +25,11 @@ from syutil.base64util import decode_base64, encode_base64 from synapse.api.errors import SynapseError, Codes from synapse.util.retryutils import get_retry_limiter - -from synapse.util.async import ObservableDeferred +from synapse.util import unwrapFirstError from OpenSSL import crypto +from collections import namedtuple import urllib import hashlib import logging @@ -38,6 +38,9 @@ import logging logger = logging.getLogger(__name__) +KeyGroup = namedtuple("KeyGroup", ("server_name", "group_id", "key_ids")) + + class Keyring(object): def __init__(self, hs): self.store = hs.get_datastore() @@ -49,141 +52,274 @@ class Keyring(object): self.key_downloads = {} - @defer.inlineCallbacks def verify_json_for_server(self, server_name, json_object): - logger.debug("Verifying for %s", server_name) - key_ids = signature_ids(json_object, server_name) - if not key_ids: - raise SynapseError( - 400, - "Not signed with a supported algorithm", - Codes.UNAUTHORIZED, - ) - try: - verify_key = yield self.get_server_verify_key(server_name, key_ids) - except IOError as e: - logger.warn( - "Got IOError when downloading keys for %s: %s %s", - server_name, type(e).__name__, str(e.message), - ) - raise SynapseError( - 502, - "Error downloading keys for %s" % (server_name,), - Codes.UNAUTHORIZED, - ) - except Exception as e: - logger.warn( - "Got Exception when downloading keys for %s: %s %s", - server_name, type(e).__name__, str(e.message), - ) - raise SynapseError( - 401, - "No key for %s with id %s" % (server_name, key_ids), - Codes.UNAUTHORIZED, - ) + return self.verify_json_objects_for_server( + [(server_name, json_object)] + )[0] - try: - verify_signed_json(json_object, server_name, verify_key) - except: - raise SynapseError( - 401, - "Invalid signature for server %s with key %s:%s" % ( - server_name, verify_key.alg, verify_key.version - ), - Codes.UNAUTHORIZED, - ) + def verify_json_objects_for_server(self, server_and_json): + """Bulk verfies signatures of json objects, bulk fetching keys as + necessary. - @defer.inlineCallbacks - def get_server_verify_key(self, server_name, key_ids): - """Finds a verification key for the server with one of the key ids. - Trys to fetch the key from a trusted perspective server first. Args: - server_name(str): The name of the server to fetch a key for. - keys_ids (list of str): The key_ids to check for. + server_and_json (list): List of pairs of (server_name, json_object) + + Returns: + list of deferreds indicating success or failure to verify each + json object's signature for the given server_name. """ - cached = yield self.store.get_server_verify_keys(server_name, key_ids) + group_id_to_json = {} + group_id_to_group = {} + group_ids = [] - if cached: - defer.returnValue(cached[0]) - return + next_group_id = 0 + deferreds = {} - download = self.key_downloads.get(server_name) + for server_name, json_object in server_and_json: + logger.debug("Verifying for %s", server_name) + group_id = next_group_id + next_group_id += 1 + group_ids.append(group_id) - if download is None: - download = self._get_server_verify_key_impl(server_name, key_ids) - download = ObservableDeferred( - download, - consumeErrors=True + key_ids = signature_ids(json_object, server_name) + if not key_ids: + deferreds[group_id] = defer.fail(SynapseError( + 400, + "Not signed with a supported algorithm", + Codes.UNAUTHORIZED, + )) + + group = KeyGroup(server_name, group_id, key_ids) + + group_id_to_group[group_id] = group + group_id_to_json[group_id] = json_object + + @defer.inlineCallbacks + def handle_key_deferred(group, deferred): + server_name = group.server_name + try: + _, _, key_id, verify_key = yield deferred + except IOError as e: + logger.warn( + "Got IOError when downloading keys for %s: %s %s", + server_name, type(e).__name__, str(e.message), + ) + raise SynapseError( + 502, + "Error downloading keys for %s" % (server_name,), + Codes.UNAUTHORIZED, + ) + except Exception as e: + logger.exception( + "Got Exception when downloading keys for %s: %s %s", + server_name, type(e).__name__, str(e.message), + ) + raise SynapseError( + 401, + "No key for %s with id %s" % (server_name, key_ids), + Codes.UNAUTHORIZED, + ) + + json_object = group_id_to_json[group.group_id] + + try: + verify_signed_json(json_object, server_name, verify_key) + except: + raise SynapseError( + 401, + "Invalid signature for server %s with key %s:%s" % ( + server_name, verify_key.alg, verify_key.version + ), + Codes.UNAUTHORIZED, + ) + + deferreds.update(self.get_server_verify_keys( + group_id_to_group + )) + + return [ + handle_key_deferred( + group_id_to_group[g_id], + deferreds[g_id], ) - self.key_downloads[server_name] = download + for g_id in group_ids + ] - @download.addBoth - def callback(ret): - del self.key_downloads[server_name] - return ret + def get_server_verify_keys(self, group_id_to_group): + """Takes a dict of KeyGroups and tries to find at least one key for + each group. + """ - r = yield download.observe() - defer.returnValue(r) + # These are functions that produce keys given a list of key ids + key_fetch_fns = ( + self.get_keys_from_store, # First try the local store + self.get_keys_from_perspectives, # Then try via perspectives + self.get_keys_from_server, # Then try directly + ) + + group_deferreds = { + group_id: defer.Deferred() + for group_id in group_id_to_group + } + + @defer.inlineCallbacks + def do_iterations(): + merged_results = {} + + missing_keys = { + group.server_name: key_id + for group in group_id_to_group.values() + for key_id in group.key_ids + } + + for fn in key_fetch_fns: + results = yield fn(missing_keys.items()) + merged_results.update(results) + + # We now need to figure out which groups we have keys for + # and which we don't + missing_groups = {} + for group in group_id_to_group.values(): + for key_id in group.key_ids: + if key_id in merged_results[group.server_name]: + group_deferreds.pop(group.group_id).callback(( + group.group_id, + group.server_name, + key_id, + merged_results[group.server_name][key_id], + )) + break + else: + missing_groups.setdefault( + group.server_name, [] + ).append(group) + + if not missing_groups: + break + + missing_keys = { + server_name: set( + key_id for group in groups for key_id in group.key_ids + ) + for server_name, groups in missing_groups.items() + } + + for group in missing_groups.values(): + group_deferreds.pop(group.group_id).errback(SynapseError( + 401, + "No key for %s with id %s" % ( + group.server_name, group.key_ids, + ), + Codes.UNAUTHORIZED, + )) + + def on_err(err): + for deferred in group_deferreds.values(): + deferred.errback(err) + group_deferreds.clear() + + do_iterations().addErrback(on_err) + + return group_deferreds @defer.inlineCallbacks - def _get_server_verify_key_impl(self, server_name, key_ids): - keys = None + def get_keys_from_store(self, server_name_and_key_ids): + res = yield defer.gatherResults( + [ + self.store.get_server_verify_keys(server_name, key_ids) + for server_name, key_ids in server_name_and_key_ids + ], + consumeErrors=True, + ).addErrback(unwrapFirstError) + defer.returnValue(dict(zip( + [server_name for server_name, _ in server_name_and_key_ids], + res + ))) + + @defer.inlineCallbacks + def get_keys_from_perspectives(self, server_name_and_key_ids): @defer.inlineCallbacks def get_key(perspective_name, perspective_keys): try: result = yield self.get_server_verify_key_v2_indirect( - server_name, key_ids, perspective_name, perspective_keys + server_name_and_key_ids, perspective_name, perspective_keys ) defer.returnValue(result) except Exception as e: - logging.info( - "Unable to getting key %r for %r from %r: %s %s", - key_ids, server_name, perspective_name, + logger.exception( + "Unable to get key from %r: %s %s", + perspective_name, type(e).__name__, str(e.message), ) + defer.returnValue({}) - perspective_results = yield defer.gatherResults([ - get_key(p_name, p_keys) - for p_name, p_keys in self.perspective_servers.items() - ]) + results = yield defer.gatherResults( + [ + get_key(p_name, p_keys) + for p_name, p_keys in self.perspective_servers.items() + ], + consumeErrors=True, + ).addErrback(unwrapFirstError) - for results in perspective_results: - if results is not None: - keys = results + union_of_keys = {} + for result in results: + for server_name, keys in result.items(): + union_of_keys.setdefault(server_name, {}).update(keys) - limiter = yield get_retry_limiter( - server_name, - self.clock, - self.store, - ) + defer.returnValue(union_of_keys) - with limiter: - if not keys: + @defer.inlineCallbacks + def get_keys_from_server(self, server_name_and_key_ids): + @defer.inlineCallbacks + def get_key(server_name, key_ids): + limiter = yield get_retry_limiter( + server_name, + self.clock, + self.store, + ) + with limiter: + keys = None try: keys = yield self.get_server_verify_key_v2_direct( server_name, key_ids ) except Exception as e: - logging.info( + logger.info( "Unable to getting key %r for %r directly: %s %s", key_ids, server_name, type(e).__name__, str(e.message), ) - if not keys: - keys = yield self.get_server_verify_key_v1_direct( - server_name, key_ids - ) + if not keys: + keys = yield self.get_server_verify_key_v1_direct( + server_name, key_ids + ) - for key_id in key_ids: - if key_id in keys: - defer.returnValue(keys[key_id]) - return - raise ValueError("No verification key found for given key ids") + keys = {server_name: keys} + + defer.returnValue(keys) + + results = yield defer.gatherResults( + [ + get_key(server_name, key_ids) + for server_name, key_ids in server_name_and_key_ids + ], + consumeErrors=True, + ).addErrback(unwrapFirstError) + + merged = {} + for result in results: + merged.update(result) + + defer.returnValue({ + server_name: keys + for server_name, keys in merged.items() + if keys + }) @defer.inlineCallbacks - def get_server_verify_key_v2_indirect(self, server_name, key_ids, + def get_server_verify_key_v2_indirect(self, server_names_and_key_ids, perspective_name, perspective_keys): limiter = yield get_retry_limiter( @@ -204,6 +340,7 @@ class Keyring(object): u"minimum_valid_until_ts": 0 } for key_id in key_ids } + for server_name, key_ids in server_names_and_key_ids } }, ) @@ -243,23 +380,29 @@ class Keyring(object): " server %r" % (perspective_name,) ) - response_keys = yield self.process_v2_response( - server_name, perspective_name, response + processed_response = yield self.process_v2_response( + perspective_name, response ) - keys.update(response_keys) + for server_name, response_keys in processed_response.items(): + keys.setdefault(server_name, {}).update(response_keys) - yield self.store_keys( - server_name=server_name, - from_server=perspective_name, - verify_keys=keys, - ) + yield defer.gatherResults( + [ + self.store_keys( + server_name=server_name, + from_server=perspective_name, + verify_keys=response_keys, + ) + for server_name, response_keys in keys.items() + ], + consumeErrors=True + ).addErrback(unwrapFirstError) defer.returnValue(keys) @defer.inlineCallbacks def get_server_verify_key_v2_direct(self, server_name, key_ids): - keys = {} for requested_key_id in key_ids: @@ -295,25 +438,30 @@ class Keyring(object): raise ValueError("TLS certificate not allowed by fingerprints") response_keys = yield self.process_v2_response( - server_name=server_name, from_server=server_name, - requested_id=requested_key_id, + requested_ids=[requested_key_id], response_json=response, ) keys.update(response_keys) - yield self.store_keys( - server_name=server_name, - from_server=server_name, - verify_keys=keys, - ) + yield defer.gatherResults( + [ + self.store_keys( + server_name=key_server_name, + from_server=server_name, + verify_keys=verify_keys, + ) + for key_server_name, verify_keys in keys.items() + ], + consumeErrors=True + ).addErrback(unwrapFirstError) defer.returnValue(keys) @defer.inlineCallbacks - def process_v2_response(self, server_name, from_server, response_json, - requested_id=None): + def process_v2_response(self, from_server, response_json, + requested_ids=[]): time_now_ms = self.clock.time_msec() response_keys = {} verify_keys = {} @@ -335,6 +483,8 @@ class Keyring(object): verify_key.time_added = time_now_ms old_verify_keys[key_id] = verify_key + results = {} + server_name = response_json["server_name"] for key_id in response_json["signatures"].get(server_name, {}): if key_id not in response_json["verify_keys"]: raise ValueError( @@ -357,28 +507,31 @@ class Keyring(object): signed_key_json_bytes = encode_canonical_json(signed_key_json) ts_valid_until_ms = signed_key_json[u"valid_until_ts"] - updated_key_ids = set() - if requested_id is not None: - updated_key_ids.add(requested_id) + updated_key_ids = set(requested_ids) updated_key_ids.update(verify_keys) updated_key_ids.update(old_verify_keys) response_keys.update(verify_keys) response_keys.update(old_verify_keys) - for key_id in updated_key_ids: - yield self.store.store_server_keys_json( - server_name=server_name, - key_id=key_id, - from_server=server_name, - ts_now_ms=time_now_ms, - ts_expires_ms=ts_valid_until_ms, - key_json_bytes=signed_key_json_bytes, - ) + yield defer.gatherResults( + [ + self.store.store_server_keys_json( + server_name=server_name, + key_id=key_id, + from_server=server_name, + ts_now_ms=time_now_ms, + ts_expires_ms=ts_valid_until_ms, + key_json_bytes=signed_key_json_bytes, + ) + for key_id in updated_key_ids + ], + consumeErrors=True, + ).addErrback(unwrapFirstError) - defer.returnValue(response_keys) + results[server_name] = response_keys - raise ValueError("No verification key found for given key ids") + defer.returnValue(results) @defer.inlineCallbacks def get_server_verify_key_v1_direct(self, server_name, key_ids): @@ -462,8 +615,13 @@ class Keyring(object): Returns: A deferred that completes when the keys are stored. """ - for key_id, key in verify_keys.items(): - # TODO(markjh): Store whether the keys have expired. - yield self.store.store_server_verify_key( - server_name, server_name, key.time_added, key - ) + # TODO(markjh): Store whether the keys have expired. + yield defer.gatherResults( + [ + self.store.store_server_verify_key( + server_name, server_name, key.time_added, key + ) + for key_id, key in verify_keys.items() + ], + consumeErrors=True, + ).addErrback(unwrapFirstError) diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 299493af9..bdfa24760 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -32,7 +32,8 @@ logger = logging.getLogger(__name__) class FederationBase(object): @defer.inlineCallbacks - def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False): + def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False, + include_none=False): """Takes a list of PDUs and checks the signatures and hashs of each one. If a PDU fails its signature check then we check if we have it in the database and if not then request if from the originating server of @@ -50,84 +51,108 @@ class FederationBase(object): Returns: Deferred : A list of PDUs that have valid signatures and hashes. """ + deferreds = self._check_sigs_and_hashes(pdus) - signed_pdus = [] + def callback(pdu): + return pdu - @defer.inlineCallbacks - def do(pdu): - try: - new_pdu = yield self._check_sigs_and_hash(pdu) - signed_pdus.append(new_pdu) - except SynapseError: - # FIXME: We should handle signature failures more gracefully. + def errback(failure, pdu): + failure.trap(SynapseError) + return None + def try_local_db(res, pdu): + if not res: # Check local db. - new_pdu = yield self.store.get_event( + return self.store.get_event( pdu.event_id, allow_rejected=True, allow_none=True, ) - if new_pdu: - signed_pdus.append(new_pdu) - return + return res - # Check pdu.origin - if pdu.origin != origin: - try: - new_pdu = yield self.get_pdu( - destinations=[pdu.origin], - event_id=pdu.event_id, - outlier=outlier, - timeout=10000, - ) - - if new_pdu: - signed_pdus.append(new_pdu) - return - except: - pass + def try_remote(res, pdu): + if not res and pdu.origin != origin: + return self.get_pdu( + destinations=[pdu.origin], + event_id=pdu.event_id, + outlier=outlier, + timeout=10000, + ).addErrback(lambda e: None) + return res + def warn(res, pdu): + if not res: logger.warn( "Failed to find copy of %s with valid signature", pdu.event_id, ) + return res - yield defer.gatherResults( - [do(pdu) for pdu in pdus], + for pdu, deferred in zip(pdus, deferreds): + deferred.addCallbacks( + callback, errback, errbackArgs=[pdu] + ).addCallback( + try_local_db, pdu + ).addCallback( + try_remote, pdu + ).addCallback( + warn, pdu + ) + + valid_pdus = yield defer.gatherResults( + deferreds, consumeErrors=True ).addErrback(unwrapFirstError) - defer.returnValue(signed_pdus) + if include_none: + defer.returnValue(valid_pdus) + else: + defer.returnValue([p for p in valid_pdus if p]) - @defer.inlineCallbacks def _check_sigs_and_hash(self, pdu): - """Throws a SynapseError if the PDU does not have the correct + return self._check_sigs_and_hashes([pdu])[0] + + def _check_sigs_and_hashes(self, pdus): + """Throws a SynapseError if a PDU does not have the correct signatures. Returns: FrozenEvent: Either the given event or it redacted if it failed the content hash check. """ - # Check signatures are correct. - redacted_event = prune_event(pdu) - redacted_pdu_json = redacted_event.get_pdu_json() - try: - yield self.keyring.verify_json_for_server( - pdu.origin, redacted_pdu_json - ) - except SynapseError: + redacted_pdus = [ + prune_event(pdu) + for pdu in pdus + ] + + deferreds = self.keyring.verify_json_objects_for_server([ + (p.origin, p.get_pdu_json()) + for p in redacted_pdus + ]) + + def callback(_, pdu, redacted): + if not check_event_content_hash(pdu): + logger.warn( + "Event content has been tampered, redacting %s: %s", + pdu.event_id, pdu.get_pdu_json() + ) + return redacted + return pdu + + def errback(failure, pdu): + failure.trap(SynapseError) logger.warn( "Signature check failed for %s", pdu.event_id, ) - raise + return failure - if not check_event_content_hash(pdu): - logger.warn( - "Event content has been tampered, redacting.", - pdu.event_id, + for deferred, pdu, redacted in zip(deferreds, pdus, redacted_pdus): + deferred.addCallbacks( + callback, errback, + callbackArgs=[pdu, redacted], + errbackArgs=[pdu], ) - defer.returnValue(redacted_event) - defer.returnValue(pdu) + return deferreds diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index d3b46b24c..7736d14fb 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -30,6 +30,7 @@ import synapse.metrics from synapse.util.retryutils import get_retry_limiter, NotRetryingDestination +import copy import itertools import logging import random @@ -167,7 +168,7 @@ class FederationClient(FederationBase): # FIXME: We should handle signature failures more gracefully. pdus[:] = yield defer.gatherResults( - [self._check_sigs_and_hash(pdu) for pdu in pdus], + self._check_sigs_and_hashes(pdus), consumeErrors=True, ).addErrback(unwrapFirstError) @@ -230,7 +231,7 @@ class FederationClient(FederationBase): pdu = pdu_list[0] # Check signatures are correct. - pdu = yield self._check_sigs_and_hash(pdu) + pdu = yield self._check_sigs_and_hashes([pdu])[0] break @@ -327,6 +328,9 @@ class FederationClient(FederationBase): @defer.inlineCallbacks def make_join(self, destinations, room_id, user_id): for destination in destinations: + if destination == self.server_name: + continue + try: ret = yield self.transport_layer.make_join( destination, room_id, user_id @@ -353,6 +357,9 @@ class FederationClient(FederationBase): @defer.inlineCallbacks def send_join(self, destinations, pdu): for destination in destinations: + if destination == self.server_name: + continue + try: time_now = self._clock.time_msec() _, content = yield self.transport_layer.send_join( @@ -374,17 +381,39 @@ class FederationClient(FederationBase): for p in content.get("auth_chain", []) ] - signed_state, signed_auth = yield defer.gatherResults( - [ - self._check_sigs_and_hash_and_fetch( - destination, state, outlier=True - ), - self._check_sigs_and_hash_and_fetch( - destination, auth_chain, outlier=True - ) - ], - consumeErrors=True - ).addErrback(unwrapFirstError) + pdus = { + p.event_id: p + for p in itertools.chain(state, auth_chain) + } + + valid_pdus = yield self._check_sigs_and_hash_and_fetch( + destination, pdus.values(), + outlier=True, + ) + + valid_pdus_map = { + p.event_id: p + for p in valid_pdus + } + + # NB: We *need* to copy to ensure that we don't have multiple + # references being passed on, as that causes... issues. + signed_state = [ + copy.copy(valid_pdus_map[p.event_id]) + for p in state + if p.event_id in valid_pdus_map + ] + + signed_auth = [ + valid_pdus_map[p.event_id] + for p in auth_chain + if p.event_id in valid_pdus_map + ] + + # NB: We *need* to copy to ensure that we don't have multiple + # references being passed on, as that causes... issues. + for s in signed_state: + s.internal_metadata = copy.deepcopy(s.internal_metadata) auth_chain.sort(key=lambda e: e.depth) @@ -396,7 +425,7 @@ class FederationClient(FederationBase): except CodeMessageException: raise except Exception as e: - logger.warn( + logger.exception( "Failed to send_join via %s: %s", destination, e.message ) diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 5bdf497b9..940a5f7e0 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from _base import SQLBaseStore +from _base import SQLBaseStore, cached from twisted.internet import defer @@ -71,6 +71,25 @@ class KeyStore(SQLBaseStore): desc="store_server_certificate", ) + @cached() + @defer.inlineCallbacks + def get_all_server_verify_keys(self, server_name): + rows = yield self._simple_select_list( + table="server_signature_keys", + keyvalues={ + "server_name": server_name, + }, + retcols=["key_id", "verify_key"], + desc="get_all_server_verify_keys", + ) + + defer.returnValue({ + row["key_id"]: decode_verify_key_bytes( + row["key_id"], str(row["verify_key"]) + ) + for row in rows + }) + @defer.inlineCallbacks def get_server_verify_keys(self, server_name, key_ids): """Retrieve the NACL verification key for a given server for the given @@ -81,24 +100,14 @@ class KeyStore(SQLBaseStore): Returns: (list of VerifyKey): The verification keys. """ - sql = ( - "SELECT key_id, verify_key FROM server_signature_keys" - " WHERE server_name = ?" - " AND key_id in (" + ",".join("?" for key_id in key_ids) + ")" - ) - - rows = yield self._execute_and_decode( - "get_server_verify_keys", sql, server_name, *key_ids - ) - - keys = [] - for row in rows: - key_id = row["key_id"] - key_bytes = row["verify_key"] - key = decode_verify_key_bytes(key_id, str(key_bytes)) - keys.append(key) - defer.returnValue(keys) + keys = yield self.get_all_server_verify_keys(server_name) + defer.returnValue({ + k: keys[k] + for k in key_ids + if k in keys and keys[k] + }) + @defer.inlineCallbacks def store_server_verify_key(self, server_name, from_server, time_now_ms, verify_key): """Stores a NACL verification key for the given server. @@ -109,7 +118,7 @@ class KeyStore(SQLBaseStore): ts_now_ms (int): The time now in milliseconds verification_key (VerifyKey): The NACL verify key. """ - return self._simple_upsert( + yield self._simple_upsert( table="server_signature_keys", keyvalues={ "server_name": server_name, @@ -123,6 +132,8 @@ class KeyStore(SQLBaseStore): desc="store_server_verify_key", ) + self.get_all_server_verify_keys.invalidate(server_name) + def store_server_keys_json(self, server_name, key_id, from_server, ts_now_ms, ts_expires_ms, key_json_bytes): """Stores the JSON bytes for a set of keys from a server @@ -152,6 +163,7 @@ class KeyStore(SQLBaseStore): "ts_valid_until_ms": ts_expires_ms, "key_json": buffer(key_json_bytes), }, + desc="store_server_keys_json", ) def get_server_keys_json(self, server_keys): From f0dd568e1681b878d9e85a12c21bfa0d5540ff73 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 26 Jun 2015 11:25:00 +0100 Subject: [PATCH 073/180] Wait for previous attempts at fetching keys for a given server before trying to fetch more --- synapse/crypto/keyring.py | 83 ++++++++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 15 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 873c9b40f..aa74d4d0c 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -27,6 +27,8 @@ from synapse.api.errors import SynapseError, Codes from synapse.util.retryutils import get_retry_limiter from synapse.util import unwrapFirstError +from synapse.util.async import ObservableDeferred + from OpenSSL import crypto from collections import namedtuple @@ -88,6 +90,8 @@ class Keyring(object): "Not signed with a supported algorithm", Codes.UNAUTHORIZED, )) + else: + deferreds[group_id] = defer.Deferred() group = KeyGroup(server_name, group_id, key_ids) @@ -133,10 +137,41 @@ class Keyring(object): Codes.UNAUTHORIZED, ) - deferreds.update(self.get_server_verify_keys( - group_id_to_group - )) + server_to_deferred = { + server_name: defer.Deferred() + for server_name, _ in server_and_json + } + # We want to wait for any previous lookups to complete before + # proceeding. + wait_on_deferred = self.wait_for_previous_lookups( + [server_name for server_name, _ in server_and_json], + server_to_deferred, + ) + + # Actually start fetching keys. + wait_on_deferred.addBoth( + lambda _: self.get_server_verify_keys(group_id_to_group, deferreds) + ) + + # When we've finished fetching all the keys for a given server_name, + # resolve the deferred passed to `wait_for_previous_lookups` so that + # any lookups waiting will proceed. + server_to_gids = {} + + def remove_deferreds(res, server_name, group_id): + server_to_gids[server_name].discard(group_id) + if not server_to_gids[server_name]: + server_to_deferred.pop(server_name).callback(None) + return res + + for g_id, deferred in deferreds.items(): + server_name = group_id_to_group[g_id].server_name + server_to_gids.setdefault(server_name, set()).add(g_id) + deferred.addBoth(remove_deferreds, server_name, g_id) + + # Pass those keys to handle_key_deferred so that the json object + # signatures can be verified return [ handle_key_deferred( group_id_to_group[g_id], @@ -145,7 +180,30 @@ class Keyring(object): for g_id in group_ids ] - def get_server_verify_keys(self, group_id_to_group): + @defer.inlineCallbacks + def wait_for_previous_lookups(self, server_names, server_to_deferred): + """Waits for any previous key lookups for the given servers to finish. + + Args: + server_names (list): list of server_names we want to lookup + server_to_deferred (dict): server_name to deferred which gets + resolved once we've finished looking up keys for that server + """ + while True: + wait_on = [ + self.key_downloads[server_name] + for server_name in server_names + if server_name in self.key_downloads + ] + if wait_on: + yield defer.DeferredList(wait_on) + else: + break + + for server_name, deferred in server_to_deferred: + self.key_downloads[server_name] = ObservableDeferred(deferred) + + def get_server_verify_keys(self, group_id_to_group, group_id_to_deferred): """Takes a dict of KeyGroups and tries to find at least one key for each group. """ @@ -157,11 +215,6 @@ class Keyring(object): self.get_keys_from_server, # Then try directly ) - group_deferreds = { - group_id: defer.Deferred() - for group_id in group_id_to_group - } - @defer.inlineCallbacks def do_iterations(): merged_results = {} @@ -182,7 +235,7 @@ class Keyring(object): for group in group_id_to_group.values(): for key_id in group.key_ids: if key_id in merged_results[group.server_name]: - group_deferreds.pop(group.group_id).callback(( + group_id_to_deferred[group.group_id].callback(( group.group_id, group.server_name, key_id, @@ -205,7 +258,7 @@ class Keyring(object): } for group in missing_groups.values(): - group_deferreds.pop(group.group_id).errback(SynapseError( + group_id_to_deferred[group.group_id].errback(SynapseError( 401, "No key for %s with id %s" % ( group.server_name, group.key_ids, @@ -214,13 +267,13 @@ class Keyring(object): )) def on_err(err): - for deferred in group_deferreds.values(): - deferred.errback(err) - group_deferreds.clear() + for deferred in group_id_to_deferred.values(): + if not deferred.called: + deferred.errback(err) do_iterations().addErrback(on_err) - return group_deferreds + return group_id_to_deferred @defer.inlineCallbacks def get_keys_from_store(self, server_name_and_key_ids): From 11374a77efda1790ab845b52294e2435b50c4e75 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 27 Jun 2015 16:59:47 +0200 Subject: [PATCH 074/180] clarify readme a bit more --- README.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 259fbaf45..21e347968 100644 --- a/README.rst +++ b/README.rst @@ -122,7 +122,10 @@ To install the synapse homeserver run:: $ pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master This installs synapse, along with the libraries it uses, into a virtual -environment under ``~/.synapse``. +environment under ``~/.synapse``. Feel free to pick a different directory +if you prefer. + +In case of problems, please see the _Troubleshooting section below. Alternatively, Silvio Fricke has contributed a Dockerfile to automate the above in Docker at https://registry.hub.docker.com/u/silviof/docker-matrix/. From 2124f668db4a7c19e3ed8357dad887711b3d3c35 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 30 Jun 2015 09:33:48 +0100 Subject: [PATCH 075/180] Add Content-Disposition headers to media repo v1 downloads --- synapse/rest/media/v1/base_resource.py | 23 ++++++++++++++++++---- synapse/rest/media/v1/download_resource.py | 8 ++++++-- synapse/rest/media/v1/upload_resource.py | 6 +++++- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 6c83a9478..04410ab82 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -30,6 +30,7 @@ from synapse.util.async import ObservableDeferred import os +import cgi import logging logger = logging.getLogger(__name__) @@ -37,7 +38,9 @@ logger = logging.getLogger(__name__) def parse_media_id(request): try: - server_name, media_id = request.postpath + # This allows users to append e.g. /test.png to the URL. Useful for + # clients that parse the URL to see content type. + server_name, media_id = request.postpath[:2] return (server_name, media_id) except: raise SynapseError( @@ -128,12 +131,19 @@ class BaseMediaResource(Resource): media_type = headers["Content-Type"][0] time_now_ms = self.clock.time_msec() + content_disposition = headers.get("Content-Disposition", None) + if content_disposition: + _, params = cgi.parse_header(content_disposition[0],) + upload_name = params.get("filename", None) + else: + upload_name = None + yield self.store.store_cached_remote_media( origin=server_name, media_id=media_id, media_type=media_type, time_now_ms=self.clock.time_msec(), - upload_name=None, + upload_name=upload_name, media_length=length, filesystem_id=file_id, ) @@ -144,7 +154,7 @@ class BaseMediaResource(Resource): media_info = { "media_type": media_type, "media_length": length, - "upload_name": None, + "upload_name": upload_name, "created_ts": time_now_ms, "filesystem_id": file_id, } @@ -157,11 +167,16 @@ class BaseMediaResource(Resource): @defer.inlineCallbacks def _respond_with_file(self, request, media_type, file_path, - file_size=None): + file_size=None, upload_name=None): logger.debug("Responding with %r", file_path) if os.path.isfile(file_path): request.setHeader(b"Content-Type", media_type.encode("UTF-8")) + if upload_name: + request.setHeader( + b"Content-Disposition", + b"inline; filename=%s" % (upload_name.encode("utf-8"),), + ) # cache for at least a day. # XXX: we might want to turn this off for data we don't want to diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py index 0fe6abf64..2af48ab62 100644 --- a/synapse/rest/media/v1/download_resource.py +++ b/synapse/rest/media/v1/download_resource.py @@ -47,10 +47,12 @@ class DownloadResource(BaseMediaResource): media_type = media_info["media_type"] media_length = media_info["media_length"] + upload_name = media_info["upload_name"] file_path = self.filepaths.local_media_filepath(media_id) yield self._respond_with_file( - request, media_type, file_path, media_length + request, media_type, file_path, media_length, + upload_name=upload_name, ) @defer.inlineCallbacks @@ -60,11 +62,13 @@ class DownloadResource(BaseMediaResource): media_type = media_info["media_type"] media_length = media_info["media_length"] filesystem_id = media_info["filesystem_id"] + upload_name = media_info["upload_name"] file_path = self.filepaths.remote_media_filepath( server_name, filesystem_id ) yield self._respond_with_file( - request, media_type, file_path, media_length + request, media_type, file_path, media_length, + upload_name=upload_name, ) diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index cc571976a..92e855a44 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -84,6 +84,10 @@ class UploadResource(BaseMediaResource): code=413, ) + upload_name = request.args.get("filename", None) + if upload_name: + upload_name = upload_name[0] + headers = request.requestHeaders if headers.hasHeader("Content-Type"): @@ -99,7 +103,7 @@ class UploadResource(BaseMediaResource): # TODO(markjh): parse content-dispostion content_uri = yield self.create_content( - media_type, None, request.content.read(), + media_type, upload_name, request.content.read(), content_length, auth_user ) From 9beaedd1642673d36428dd796dda62f18a937c2a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 30 Jun 2015 10:31:59 +0100 Subject: [PATCH 076/180] Enforce ascii filenames for uploads --- synapse/rest/media/v1/base_resource.py | 3 +++ synapse/rest/media/v1/upload_resource.py | 4 +++- synapse/util/stringutils.py | 9 +++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 04410ab82..1b7517e2f 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -27,6 +27,7 @@ from twisted.web.resource import Resource from twisted.protocols.basic import FileSender from synapse.util.async import ObservableDeferred +from synapse.util.stringutils import is_ascii import os @@ -135,6 +136,8 @@ class BaseMediaResource(Resource): if content_disposition: _, params = cgi.parse_header(content_disposition[0],) upload_name = params.get("filename", None) + if upload_name and not is_ascii(upload_name): + upload_name = None else: upload_name = None diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 92e855a44..cdd1d44e0 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -15,7 +15,7 @@ from synapse.http.server import respond_with_json, request_handler -from synapse.util.stringutils import random_string +from synapse.util.stringutils import random_string, is_ascii from synapse.api.errors import SynapseError from twisted.web.server import NOT_DONE_YET @@ -87,6 +87,8 @@ class UploadResource(BaseMediaResource): upload_name = request.args.get("filename", None) if upload_name: upload_name = upload_name[0] + if upload_name and not is_ascii(upload_name): + raise SynapseError(400, "filename must be ascii") headers = request.requestHeaders diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py index 52e66beae..7a1e96af3 100644 --- a/synapse/util/stringutils.py +++ b/synapse/util/stringutils.py @@ -33,3 +33,12 @@ def random_string_with_symbols(length): return ''.join( random.choice(_string_with_symbols) for _ in xrange(length) ) + + +def is_ascii(s): + try: + s.encode("ascii") + except UnicodeDecodeError: + return False + else: + return True From 901f56fa63c0f8cd4519e2a2f774724af9ea3431 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 30 Jun 2015 15:29:47 +0100 Subject: [PATCH 077/180] Add tables for receipts --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/21/receipts.sql | 35 ++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/21/receipts.sql diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c137f4782..275598add 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 20 +SCHEMA_VERSION = 21 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/21/receipts.sql b/synapse/storage/schema/delta/21/receipts.sql new file mode 100644 index 000000000..da9e18e90 --- /dev/null +++ b/synapse/storage/schema/delta/21/receipts.sql @@ -0,0 +1,35 @@ +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CREATE TABLE IF NOT EXISTS receipts_graph( + room_id TEXT NOT NULL, + receipt_type TEXT NOT NULL, + user_id TEXT NOT NULL, + event_id TEXT NOT NULL +); + +CREATE INDEX receipts_graph_room_tuple ON receipts_graph( + room_id, receipt_type, user_id +); + +CREATE TABLE IF NOT EXISTS receipts_linearized ( + room_id TEXT NOT NULL, + receipt_type TEXT NOT NULL, + user_id TEXT NOT NULL, + event_id TEXT NOT NULL +); + +CREATE INDEX receipts_graph_room_tuple ON receipts_graph( + room_id, receipt_type, user_id +); From 480d7203882c71fdee15d464766ba459ac065133 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 1 Jul 2015 15:12:32 +0100 Subject: [PATCH 078/180] Bump changelog and version to v0.9.3 --- CHANGES.rst | 7 ++++++- synapse/__init__.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index caebde334..6a5fce899 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,11 @@ -Changes in synapse v0.9.3 (2015-XX-XX) +Changes in synapse v0.9.3 (2015-07-01) ====================================== +No changes from v0.9.3 Release Candidate 1. + +Changes in synapse v0.9.3-rc1 (2015-06-23) +========================================== + General: * Fix a memory leak in the notifier. (SYN-412) diff --git a/synapse/__init__.py b/synapse/__init__.py index 1f09df79c..96e37308d 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.9.3-rc1" +__version__ = "0.9.3" From 80a61330ee794147b213b1d54f2292a1c9adc002 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 1 Jul 2015 11:41:55 +0100 Subject: [PATCH 079/180] Add basic storage functions for handling of receipts --- synapse/storage/_base.py | 3 +- synapse/storage/receipts.py | 162 +++++++++++++++++++ synapse/storage/schema/delta/21/receipts.sql | 31 ++-- synapse/storage/util/id_generators.py | 7 +- 4 files changed, 186 insertions(+), 17 deletions(-) create mode 100644 synapse/storage/receipts.py diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 8d33def6c..8f812f0fd 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -329,13 +329,14 @@ class SQLBaseStore(object): self.database_engine = hs.database_engine - self._stream_id_gen = StreamIdGenerator() + self._stream_id_gen = StreamIdGenerator("events", "stream_ordering") self._transaction_id_gen = IdGenerator("sent_transactions", "id", self) self._state_groups_id_gen = IdGenerator("state_groups", "id", self) self._access_tokens_id_gen = IdGenerator("access_tokens", "id", self) self._pushers_id_gen = IdGenerator("pushers", "id", self) self._push_rule_id_gen = IdGenerator("push_rules", "id", self) self._push_rules_enable_id_gen = IdGenerator("push_rules_enable", "id", self) + self._receipts_id_gen = StreamIdGenerator("receipts_linearized", "stream_id") def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py new file mode 100644 index 000000000..0168e74a0 --- /dev/null +++ b/synapse/storage/receipts.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- +# Copyright 2014, 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import SQLBaseStore, cached + +from twisted.internet import defer + + +class ReceiptStore(SQLBaseStore): + + @cached + @defer.inlineCallbacks + def get_linearized_receipts_for_room(self, room_id): + rows = yield self._simple_select_list( + table="receipts_linearized", + keyvalues={"room_id": room_id}, + retcols=["receipt_type", "user_id", "event_id"], + desc="get_linearized_receipts_for_room", + ) + + result = {} + for row in rows: + result.setdefault( + row["event_id"], {} + ).setdefault( + row["receipt_type"], [] + ).append(row["user_id"]) + + defer.returnValue(result) + + @cached + @defer.inlineCallbacks + def get_graph_receipts_for_room(self, room_id): + rows = yield self._simple_select_list( + table="receipts_graph", + keyvalues={"room_id": room_id}, + retcols=["receipt_type", "user_id", "event_id"], + desc="get_linearized_receipts_for_room", + ) + + result = {} + for row in rows: + result.setdefault( + row["user_id"], {} + ).setdefault( + row["receipt_type"], [] + ).append(row["event_id"]) + + defer.returnValue(result) + + def insert_linearized_receipt_txn(self, txn, room_id, receipt_type, + user_id, event_id, stream_id): + self._simple_delete_txn( + txn, + table="receipts_linearized", + keyvalues={ + "stream_id": stream_id, + "room_id": room_id, + "receipt_type": receipt_type, + "user_id": user_id, + } + ) + + self._simple_insert_txn( + txn, + table="receipts_linearized", + values={ + "room_id": room_id, + "receipt_type": receipt_type, + "user_id": user_id, + "event_id": event_id, + } + ) + + @defer.inlineCallbacks + def insert_receipt(self, room_id, receipt_type, user_id, event_ids): + if not event_ids: + return + + if len(event_ids) == 1: + linearized_event_id = event_ids[0] + else: + # we need to points in graph -> linearized form. + def graph_to_linear(txn): + query = ( + "SELECT event_id WHERE room_id = ? AND stream_ordering IN (" + " SELECT max(stream_ordering) WHERE event_id IN (%s)" + ")" + ) % (",".join(["?"] * len(event_ids))) + + txn.execute(query, [room_id] + event_ids) + rows = txn.fetchall() + if rows: + return rows[0][0] + else: + # TODO: ARGH?! + return None + + linearized_event_id = yield self.runInteraction( + graph_to_linear, desc="insert_receipt_conv" + ) + + stream_id_manager = yield self._stream_id_gen.get_next(self) + with stream_id_manager() as stream_id: + yield self.runInteraction( + self.insert_linearized_receipt_txn, + room_id, receipt_type, user_id, linearized_event_id, + stream_id=stream_id, + desc="insert_linearized_receipt" + ) + + yield self.insert_graph_receipt( + room_id, receipt_type, user_id, event_ids + ) + + max_persisted_id = yield self._stream_id_gen.get_max_token(self) + defer.returnValue((stream_id, max_persisted_id)) + + def insert_graph_receipt(self, room_id, receipt_type, + user_id, event_ids): + return self.runInteraction( + self.insert_graph_receipt_txn, + room_id, receipt_type, user_id, event_ids, + desc="insert_graph_receipt" + ) + + def insert_graph_receipt_txn(self, txn, room_id, receipt_type, + user_id, event_ids): + self._simple_delete_txn( + txn, + table="receipts_graph", + keyvalues={ + "room_id": room_id, + "receipt_type": receipt_type, + "user_id": user_id, + } + ) + self._simple_insert_many_txn( + txn, + table="receipts_graph", + values=[ + { + "room_id": room_id, + "receipt_type": receipt_type, + "user_id": user_id, + "event_id": event_id, + } + for event_id in event_ids + ], + ) diff --git a/synapse/storage/schema/delta/21/receipts.sql b/synapse/storage/schema/delta/21/receipts.sql index da9e18e90..ccd64ec7f 100644 --- a/synapse/storage/schema/delta/21/receipts.sql +++ b/synapse/storage/schema/delta/21/receipts.sql @@ -1,16 +1,18 @@ -# Copyright 2015 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + CREATE TABLE IF NOT EXISTS receipts_graph( room_id TEXT NOT NULL, @@ -24,12 +26,13 @@ CREATE INDEX receipts_graph_room_tuple ON receipts_graph( ); CREATE TABLE IF NOT EXISTS receipts_linearized ( + stream_id BIGINT NOT NULL, room_id TEXT NOT NULL, receipt_type TEXT NOT NULL, user_id TEXT NOT NULL, event_id TEXT NOT NULL ); -CREATE INDEX receipts_graph_room_tuple ON receipts_graph( +CREATE INDEX receipts_linearized_room_tuple ON receipts_graph( room_id, receipt_type, user_id ); diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index 89d1643f1..b39006315 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -72,7 +72,10 @@ class StreamIdGenerator(object): with stream_id_gen.get_next_txn(txn) as stream_id: # ... persist event ... """ - def __init__(self): + def __init__(self, table, column): + self.table = table + self.column = column + self._lock = threading.Lock() self._current_max = None @@ -126,7 +129,7 @@ class StreamIdGenerator(object): def _get_or_compute_current_max(self, txn): with self._lock: - txn.execute("SELECT MAX(stream_ordering) FROM events") + txn.execute("SELECT MAX(%s) FROM %s" % (self.column, self.table)) rows = txn.fetchall() val, = rows[0] From 0862fed2a89723e8aa6a4df9f1dbad975a7fbffc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 1 Jul 2015 17:19:31 +0100 Subject: [PATCH 080/180] Add basic ReceiptHandler --- synapse/handlers/receipts.py | 130 +++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 synapse/handlers/receipts.py diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py new file mode 100644 index 000000000..f3f705063 --- /dev/null +++ b/synapse/handlers/receipts.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Contains handlers for federation events.""" + +from ._base import BaseHandler + +from twisted.internet import defer + +from synapse.util.logcontext import PreserveLoggingContext + +import logging + + +logger = logging.getLogger(__name__) + + +class ReceiptsHandler(BaseHandler): + def __init__(self, hs): + super(ReceiptsHandler, self).__init__(hs) + + self.federation.register_edu_handler( + "m.receipt", self._received_remote_receipt + ) + + self._latest_serial = 0 + + @defer.inlineCallbacks + def received_client_receipt(self, room_id, receipt_type, user_id, + event_id): + # 1. Persist. + # 2. Notify local clients + # 3. Notify remote servers + + receipt = { + "room_id": room_id, + "receipt_type": receipt_type, + "user_id": user_id, + "event_ids": [event_id], + } + + yield self._handle_new_receipts([receipt]) + self._push_remotes([receipt]) + + @defer.inlineCallbacks + def _received_remote_receipt(self, origin, content): + receipts = [ + { + "room_id": room_id, + "receipt_type": receipt_type, + "user_id": user_id, + "event_ids": [event_id], + } + for room_id, room_values in content.items() + for event_id, ev_values in room_values.items() + for receipt_type, users in ev_values.items() + for user_id in users + ] + + yield self._handle_new_receipts(receipts) + + @defer.inlineCallbacks + def _handle_new_receipts(self, receipts): + for receipt in receipts: + room_id = receipt["room_id"] + receipt_type = receipt["receipt_type"] + user_id = receipt["user_id"] + event_ids = receipt["event_ids"] + + stream_id, max_persisted_id = yield self.store.insert_receipt( + room_id, receipt_type, user_id, event_ids, + ) + + # TODO: Use max_persisted_id + + self._latest_serial = max(self._latest_serial, stream_id) + + with PreserveLoggingContext(): + self.notifier.on_new_user_event( + "recei[t_key", self._latest_serial, rooms=[room_id] + ) + + localusers = set() + remotedomains = set() + + rm_handler = self.homeserver.get_handlers().room_member_handler + yield rm_handler.fetch_room_distributions_into( + room_id, localusers=localusers, remotedomains=remotedomains + ) + + receipt["remotedomains"] = remotedomains + + self.notifier.on_new_user_event( + "receipt_key", self._latest_room_serial, rooms=[room_id] + ) + + def _push_remotes(self, receipts): + # TODO: Some of this stuff should be coallesced. + for receipt in receipts: + room_id = receipt["room_id"] + receipt_type = receipt["receipt_type"] + user_id = receipt["user_id"] + event_ids = receipt["event_ids"] + remotedomains = receipt["remotedomains"] + + for domain in remotedomains: + self.federation.send_edu( + destination=domain, + edu_type="m.receipt", + content={ + room_id: { + event_id: { + receipt_type: [user_id] + } + for event_id in event_ids + }, + }, + ) From ddf7979531cf514ee54e280714bdf228d88a62d5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 2 Jul 2015 11:40:22 +0100 Subject: [PATCH 081/180] Add receipts_key to StreamToken --- synapse/notifier.py | 2 +- synapse/streams/events.py | 3 ++- synapse/types.py | 6 +++++- tests/rest/client/v1/test_presence.py | 4 ++-- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index bdd03dcbe..f13164dbd 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -283,7 +283,7 @@ class Notifier(object): @defer.inlineCallbacks def wait_for_events(self, user, rooms, timeout, callback, - from_token=StreamToken("s0", "0", "0")): + from_token=StreamToken("s0", "0", "0", "0")): """Wait until the callback returns a non empty response or the timeout fires. """ diff --git a/synapse/streams/events.py b/synapse/streams/events.py index dff7970be..0a1a3a3d0 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -62,7 +62,8 @@ class EventSources(object): ), typing_key=( yield self.sources["typing"].get_current_key() - ) + ), + receipt_key="0", ) defer.returnValue(token) diff --git a/synapse/types.py b/synapse/types.py index 1b21160c5..dd1b10d64 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -100,7 +100,7 @@ class EventID(DomainSpecificString): class StreamToken( namedtuple( "Token", - ("room_key", "presence_key", "typing_key") + ("room_key", "presence_key", "typing_key", "receipt_key") ) ): _SEPARATOR = "_" @@ -109,6 +109,9 @@ class StreamToken( def from_string(cls, string): try: keys = string.split(cls._SEPARATOR) + if len(keys) == len(cls._fields) - 1: + # i.e. old token from before receipt_key + keys.append("0") return cls(*keys) except: raise SynapseError(400, "Invalid Token") @@ -131,6 +134,7 @@ class StreamToken( (other_token.room_stream_id < self.room_stream_id) or (int(other_token.presence_key) < int(self.presence_key)) or (int(other_token.typing_key) < int(self.typing_key)) + or (int(other_token.receipt_key) < int(self.receipt_key)) ) def copy_and_advance(self, key, new_value): diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 4b32c7a20..089a71568 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -357,7 +357,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): # all be ours # I'll already get my own presence state change - self.assertEquals({"start": "0_1_0", "end": "0_1_0", "chunk": []}, + self.assertEquals({"start": "0_1_0_0", "end": "0_1_0_0", "chunk": []}, response ) @@ -376,7 +376,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): "/events?from=s0_1_0&timeout=0", None) self.assertEquals(200, code) - self.assertEquals({"start": "s0_1_0", "end": "s0_2_0", "chunk": [ + self.assertEquals({"start": "s0_1_0_0", "end": "s0_2_0_0", "chunk": [ {"type": "m.presence", "content": { "user_id": "@banana:test", From bd1236c0ee5b3703de51dc773a02da92e0960d0f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 2 Jul 2015 11:40:56 +0100 Subject: [PATCH 082/180] Consolidate duplicate code in notifier --- synapse/handlers/presence.py | 2 +- synapse/handlers/receipts.py | 4 ++-- synapse/handlers/typing.py | 2 +- synapse/notifier.py | 35 +++++++++++------------------------ tests/handlers/test_typing.py | 20 ++++++++++---------- 5 files changed, 25 insertions(+), 38 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 7c0319831..341a516da 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -992,7 +992,7 @@ class PresenceHandler(BaseHandler): room_ids([str]): List of room_ids to notify. """ with PreserveLoggingContext(): - self.notifier.on_new_user_event( + self.notifier.on_new_event( "presence_key", self._user_cachemap_latest_serial, users_to_push, diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index f3f705063..f0d12d35f 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -88,7 +88,7 @@ class ReceiptsHandler(BaseHandler): self._latest_serial = max(self._latest_serial, stream_id) with PreserveLoggingContext(): - self.notifier.on_new_user_event( + self.notifier.on_new_event( "recei[t_key", self._latest_serial, rooms=[room_id] ) @@ -102,7 +102,7 @@ class ReceiptsHandler(BaseHandler): receipt["remotedomains"] = remotedomains - self.notifier.on_new_user_event( + self.notifier.on_new_event( "receipt_key", self._latest_room_serial, rooms=[room_id] ) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index a9895292c..026bd2b9d 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -218,7 +218,7 @@ class TypingNotificationHandler(BaseHandler): self._room_serials[room_id] = self._latest_room_serial with PreserveLoggingContext(): - self.notifier.on_new_user_event( + self.notifier.on_new_event( "typing_key", self._latest_room_serial, rooms=[room_id] ) diff --git a/synapse/notifier.py b/synapse/notifier.py index f13164dbd..85ae34313 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -221,16 +221,7 @@ class Notifier(object): event ) - room_id = event.room_id - - room_user_streams = self.room_to_user_streams.get(room_id, set()) - - user_streams = room_user_streams.copy() - - for user in extra_users: - user_stream = self.user_to_user_stream.get(str(user)) - if user_stream is not None: - user_streams.add(user_stream) + app_streams = set() for appservice in self.appservice_to_user_streams: # TODO (kegan): Redundant appservice listener checks? @@ -242,24 +233,20 @@ class Notifier(object): app_user_streams = self.appservice_to_user_streams.get( appservice, set() ) - user_streams |= app_user_streams + app_streams |= app_user_streams - logger.debug("on_new_room_event listeners %s", user_streams) - - time_now_ms = self.clock.time_msec() - for user_stream in user_streams: - try: - user_stream.notify( - "room_key", "s%d" % (room_stream_id,), time_now_ms - ) - except: - logger.exception("Failed to notify listener") + self.on_new_event( + "room_key", room_stream_id, + users=extra_users, + rooms=[event.room_id], + extra_streams=app_streams, + ) @defer.inlineCallbacks @log_function - def on_new_user_event(self, stream_key, new_token, users=[], rooms=[]): - """ Used to inform listeners that something has happend - presence/user event wise. + def on_new_event(self, stream_key, new_token, users=[], rooms=[], + extra_streams=set()): + """ Used to inform listeners that something has happend event wise. Will wake up all listeners for the given users and rooms. """ diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 7ccbe2ea9..41bb08b7c 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -66,8 +66,8 @@ class TypingNotificationsTestCase(unittest.TestCase): self.mock_federation_resource = MockHttpResource() - mock_notifier = Mock(spec=["on_new_user_event"]) - self.on_new_user_event = mock_notifier.on_new_user_event + mock_notifier = Mock(spec=["on_new_event"]) + self.on_new_event = mock_notifier.on_new_event self.auth = Mock(spec=[]) @@ -182,7 +182,7 @@ class TypingNotificationsTestCase(unittest.TestCase): timeout=20000, ) - self.on_new_user_event.assert_has_calls([ + self.on_new_event.assert_has_calls([ call('typing_key', 1, rooms=[self.room_id]), ]) @@ -245,7 +245,7 @@ class TypingNotificationsTestCase(unittest.TestCase): ) ) - self.on_new_user_event.assert_has_calls([ + self.on_new_event.assert_has_calls([ call('typing_key', 1, rooms=[self.room_id]), ]) @@ -299,7 +299,7 @@ class TypingNotificationsTestCase(unittest.TestCase): room_id=self.room_id, ) - self.on_new_user_event.assert_has_calls([ + self.on_new_event.assert_has_calls([ call('typing_key', 1, rooms=[self.room_id]), ]) @@ -331,10 +331,10 @@ class TypingNotificationsTestCase(unittest.TestCase): timeout=10000, ) - self.on_new_user_event.assert_has_calls([ + self.on_new_event.assert_has_calls([ call('typing_key', 1, rooms=[self.room_id]), ]) - self.on_new_user_event.reset_mock() + self.on_new_event.reset_mock() self.assertEquals(self.event_source.get_current_key(), 1) events = yield self.event_source.get_new_events_for_user(self.u_apple, 0, None) @@ -351,7 +351,7 @@ class TypingNotificationsTestCase(unittest.TestCase): self.clock.advance_time(11) - self.on_new_user_event.assert_has_calls([ + self.on_new_event.assert_has_calls([ call('typing_key', 2, rooms=[self.room_id]), ]) @@ -377,10 +377,10 @@ class TypingNotificationsTestCase(unittest.TestCase): timeout=10000, ) - self.on_new_user_event.assert_has_calls([ + self.on_new_event.assert_has_calls([ call('typing_key', 3, rooms=[self.room_id]), ]) - self.on_new_user_event.reset_mock() + self.on_new_event.reset_mock() self.assertEquals(self.event_source.get_current_key(), 3) events = yield self.event_source.get_new_events_for_user(self.u_apple, 0, None) From ac78e60de6fd2e409d6abd806e8850d539bbe644 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 2 Jul 2015 13:18:41 +0100 Subject: [PATCH 083/180] Add stream_id index --- synapse/storage/schema/delta/21/receipts.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/storage/schema/delta/21/receipts.sql b/synapse/storage/schema/delta/21/receipts.sql index ccd64ec7f..ac7738e37 100644 --- a/synapse/storage/schema/delta/21/receipts.sql +++ b/synapse/storage/schema/delta/21/receipts.sql @@ -33,6 +33,10 @@ CREATE TABLE IF NOT EXISTS receipts_linearized ( event_id TEXT NOT NULL ); -CREATE INDEX receipts_linearized_room_tuple ON receipts_graph( +CREATE INDEX receipts_linearized_room_tuple ON receipts_linearized( room_id, receipt_type, user_id ); + +CREATE INDEX receipts_linearized_id ON receipts_linearized( + stream_id +); From 1a605456260bfb46d8bb9cff2d40d19aec03daa4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 2 Jul 2015 16:20:10 +0100 Subject: [PATCH 084/180] Add basic impl for room history ACL on GET /messages client API --- synapse/api/constants.py | 2 ++ synapse/handlers/message.py | 33 ++++++++++++++++++- synapse/storage/state.py | 63 +++++++++++++++++++++++++++++++++++-- 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index d8a18ee87..3e15e8a9d 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -75,6 +75,8 @@ class EventTypes(object): Redaction = "m.room.redaction" Feedback = "m.room.message.feedback" + RoomHistoryVisibility = "m.room.history_visibility" + # These are used for validation Message = "m.room.message" Topic = "m.room.topic" diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index e324662f1..17c75f33c 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -113,11 +113,42 @@ class MessageHandler(BaseHandler): "room_key", next_key ) + if not events: + defer.returnValue({ + "chunk": [], + "start": pagin_config.from_token.to_string(), + "end": next_token.to_string(), + }) + + states = yield self.store.get_state_for_events( + room_id, [e.event_id for e in events], + ) + + events_and_states = zip(events, states) + + def allowed(event_and_state): + _, state = event_and_state + + membership = state.get((EventTypes.Member, user_id), None) + if membership and membership.membership == Membership.JOIN: + return True + + history = state.get((EventTypes.RoomHistoryVisibility, ''), None) + if history and history.content["visibility"] == "after_join": + return False + + events_and_states = filter(allowed, events_and_states) + events = [ + ev + for ev, _ in events_and_states + ] + time_now = self.clock.time_msec() chunk = { "chunk": [ - serialize_event(e, time_now, as_client_event) for e in events + serialize_event(e, time_now, as_client_event) + for e in events ], "start": pagin_config.from_token.to_string(), "end": next_token.to_string(), diff --git a/synapse/storage/state.py b/synapse/storage/state.py index f2b17f29e..d7844edee 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -92,11 +92,11 @@ class StateStore(SQLBaseStore): defer.returnValue(dict(state_list)) @cached(num_args=1) - def _fetch_events_for_group(self, state_group, events): + def _fetch_events_for_group(self, key, events): return self._get_events( events, get_prev_content=False ).addCallback( - lambda evs: (state_group, evs) + lambda evs: (key, evs) ) def _store_state_groups_txn(self, txn, event, context): @@ -194,6 +194,65 @@ class StateStore(SQLBaseStore): events = yield self._get_events(event_ids, get_prev_content=False) defer.returnValue(events) + @defer.inlineCallbacks + def get_state_for_events(self, room_id, event_ids): + def f(txn): + groups = set() + event_to_group = {} + for event_id in event_ids: + # TODO: Remove this loop. + group = self._simple_select_one_onecol_txn( + txn, + table="event_to_state_groups", + keyvalues={"event_id": event_id}, + retcol="state_group", + allow_none=True, + ) + if group: + event_to_group[event_id] = group + groups.add(group) + + group_to_state_ids = {} + for group in groups: + state_ids = self._simple_select_onecol_txn( + txn, + table="state_groups_state", + keyvalues={"state_group": group}, + retcol="event_id", + ) + + group_to_state_ids[group] = state_ids + + return event_to_group, group_to_state_ids + + res = yield self.runInteraction( + "annotate_events_with_state_groups", + f, + ) + + event_to_group, group_to_state_ids = res + + state_list = yield defer.gatherResults( + [ + self._fetch_events_for_group(group, vals) + for group, vals in group_to_state_ids.items() + ], + consumeErrors=True, + ) + + state_dict = { + group: { + (ev.type, ev.state_key): ev + for ev in state + } + for group, state in state_list + } + + defer.returnValue([ + state_dict.get(event_to_group.get(event, None), None) + for event in event_ids + ]) + def _make_group_id(clock): return str(int(clock.time_msec())) + random_string(5) From 41938afed884361451ad6e91eb44e805ebbdaeb0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 2 Jul 2015 17:02:10 +0100 Subject: [PATCH 085/180] Make v1 initial syncs respect room history ACL --- synapse/handlers/message.py | 61 ++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 22 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 17c75f33c..00c7dbec8 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -120,28 +120,7 @@ class MessageHandler(BaseHandler): "end": next_token.to_string(), }) - states = yield self.store.get_state_for_events( - room_id, [e.event_id for e in events], - ) - - events_and_states = zip(events, states) - - def allowed(event_and_state): - _, state = event_and_state - - membership = state.get((EventTypes.Member, user_id), None) - if membership and membership.membership == Membership.JOIN: - return True - - history = state.get((EventTypes.RoomHistoryVisibility, ''), None) - if history and history.content["visibility"] == "after_join": - return False - - events_and_states = filter(allowed, events_and_states) - events = [ - ev - for ev, _ in events_and_states - ] + events = yield self._filter_events_for_client(user_id, room_id, events) time_now = self.clock.time_msec() @@ -156,6 +135,36 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) + @defer.inlineCallbacks + def _filter_events_for_client(self, user_id, room_id, events): + states = yield self.store.get_state_for_events( + room_id, [e.event_id for e in events], + ) + + events_and_states = zip(events, states) + + def allowed(event_and_state): + event, state = event_and_state + + if event.type == EventTypes.RoomHistoryVisibility: + return True + + membership = state.get((EventTypes.Member, user_id), None) + if membership and membership.membership == Membership.JOIN: + return True + + history = state.get((EventTypes.RoomHistoryVisibility, ''), None) + if history and history.content.get("visibility", None) == "after_join": + return False + + return True + + events_and_states = filter(allowed, events_and_states) + defer.returnValue([ + ev + for ev, _ in events_and_states + ]) + @defer.inlineCallbacks def create_and_send_event(self, event_dict, ratelimit=True, client=None, txn_id=None): @@ -347,6 +356,10 @@ class MessageHandler(BaseHandler): ] ).addErrback(unwrapFirstError) + messages = yield self._filter_events_for_client( + user_id, event.room_id, messages + ) + start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) time_now = self.clock.time_msec() @@ -448,6 +461,10 @@ class MessageHandler(BaseHandler): consumeErrors=True, ).addErrback(unwrapFirstError) + messages = yield self._filter_events_for_client( + user_id, room_id, messages + ) + start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) From 00ab882ed66d1ecdbd9de15c7ac591f4ee07b9f8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 3 Jul 2015 10:31:17 +0100 Subject: [PATCH 086/180] Add m.room.history_visibility to list of auth events --- synapse/api/auth.py | 2 +- synapse/events/utils.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 4da62e5d8..deca747f7 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -29,7 +29,7 @@ logger = logging.getLogger(__name__) AuthEventTypes = ( EventTypes.Create, EventTypes.Member, EventTypes.PowerLevels, - EventTypes.JoinRules, + EventTypes.JoinRules, EventTypes.RoomHistoryVisibility, ) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 1aa952150..4c82780f4 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -74,6 +74,8 @@ def prune_event(event): ) elif event_type == EventTypes.Aliases: add_fields("aliases") + elif event_type == EventTypes.RoomHistoryVisibility: + add_fields("visibility") allowed_fields = { k: v From 12b83f1a0d1e666b8dc629358a904613b23aac11 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 3 Jul 2015 11:24:55 +0100 Subject: [PATCH 087/180] If user supplies filename in URL when downloading from media repo, use that name in Content Disposition --- synapse/rest/media/v1/base_resource.py | 5 ++++- synapse/rest/media/v1/download_resource.py | 16 +++++++++------- synapse/rest/media/v1/thumbnail_resource.py | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 1b7517e2f..c43ae0314 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -42,7 +42,10 @@ def parse_media_id(request): # This allows users to append e.g. /test.png to the URL. Useful for # clients that parse the URL to see content type. server_name, media_id = request.postpath[:2] - return (server_name, media_id) + if len(request.postpath) > 2 and is_ascii(request.postpath[-1]): + return server_name, media_id, request.postpath[-1] + else: + return server_name, media_id, None except: raise SynapseError( 404, diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py index 2af48ab62..ab384e538 100644 --- a/synapse/rest/media/v1/download_resource.py +++ b/synapse/rest/media/v1/download_resource.py @@ -32,14 +32,16 @@ class DownloadResource(BaseMediaResource): @request_handler @defer.inlineCallbacks def _async_render_GET(self, request): - server_name, media_id = parse_media_id(request) + server_name, media_id, name = parse_media_id(request) if server_name == self.server_name: - yield self._respond_local_file(request, media_id) + yield self._respond_local_file(request, media_id, name) else: - yield self._respond_remote_file(request, server_name, media_id) + yield self._respond_remote_file( + request, server_name, media_id, name + ) @defer.inlineCallbacks - def _respond_local_file(self, request, media_id): + def _respond_local_file(self, request, media_id, name): media_info = yield self.store.get_local_media(media_id) if not media_info: self._respond_404(request) @@ -47,7 +49,7 @@ class DownloadResource(BaseMediaResource): media_type = media_info["media_type"] media_length = media_info["media_length"] - upload_name = media_info["upload_name"] + upload_name = name if name else media_info["upload_name"] file_path = self.filepaths.local_media_filepath(media_id) yield self._respond_with_file( @@ -56,13 +58,13 @@ class DownloadResource(BaseMediaResource): ) @defer.inlineCallbacks - def _respond_remote_file(self, request, server_name, media_id): + def _respond_remote_file(self, request, server_name, media_id, name): media_info = yield self._get_remote_media(server_name, media_id) media_type = media_info["media_type"] media_length = media_info["media_length"] filesystem_id = media_info["filesystem_id"] - upload_name = media_info["upload_name"] + upload_name = name if name else media_info["upload_name"] file_path = self.filepaths.remote_media_filepath( server_name, filesystem_id diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 1dadd880b..4a9b6d8ee 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -36,7 +36,7 @@ class ThumbnailResource(BaseMediaResource): @request_handler @defer.inlineCallbacks def _async_render_GET(self, request): - server_name, media_id = parse_media_id(request) + server_name, media_id, _ = parse_media_id(request) width = parse_integer(request, "width") height = parse_integer(request, "height") method = parse_string(request, "method", "scale") From 400894616d15a01c168b2356d950972b6e746496 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 3 Jul 2015 14:51:01 +0100 Subject: [PATCH 088/180] Respect m.room.history_visibility in v2_alpha sync API --- synapse/handlers/sync.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index bd8c60368..5078c4e45 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -292,6 +292,36 @@ class SyncHandler(BaseHandler): next_batch=now_token, )) + @defer.inlineCallbacks + def _filter_events_for_client(self, user_id, room_id, events): + states = yield self.store.get_state_for_events( + room_id, [e.event_id for e in events], + ) + + events_and_states = zip(events, states) + + def allowed(event_and_state): + event, state = event_and_state + + if event.type == EventTypes.RoomHistoryVisibility: + return True + + membership = state.get((EventTypes.Member, user_id), None) + if membership and membership.membership == Membership.JOIN: + return True + + history = state.get((EventTypes.RoomHistoryVisibility, ''), None) + if history and history.content.get("visibility", None) == "after_join": + return False + + return True + + events_and_states = filter(allowed, events_and_states) + defer.returnValue([ + ev + for ev, _ in events_and_states + ]) + @defer.inlineCallbacks def load_filtered_recents(self, room_id, sync_config, now_token, since_token=None): @@ -313,6 +343,9 @@ class SyncHandler(BaseHandler): (room_key, _) = keys end_key = "s" + room_key.split('-')[-1] loaded_recents = sync_config.filter.filter_room_events(events) + loaded_recents = yield self._filter_events_for_client( + sync_config.user.to_string(), room_id, loaded_recents, + ) loaded_recents.extend(recents) recents = loaded_recents if len(events) <= load_limit: From c3e2600c6727534d4ebf20dcd8219e248ca31461 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 3 Jul 2015 17:52:57 +0100 Subject: [PATCH 089/180] Filter and redact events that the other server doesn't have permission to see during backfill --- synapse/handlers/federation.py | 44 ++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b5d882fd6..663d05c63 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -31,6 +31,8 @@ from synapse.crypto.event_signing import ( ) from synapse.types import UserID +from synapse.events.utils import prune_event + from synapse.util.retryutils import NotRetryingDestination from twisted.internet import defer @@ -222,6 +224,46 @@ class FederationHandler(BaseHandler): "user_joined_room", user=user, room_id=event.room_id ) + @defer.inlineCallbacks + def _filter_events_for_server(self, server_name, room_id, events): + states = yield self.store.get_state_for_events( + room_id, [e.event_id for e in events], + ) + + events_and_states = zip(events, states) + + def redact_disallowed(event_and_state): + event, state = event_and_state + + if not state: + return event + + history = state.get((EventTypes.RoomHistoryVisibility, ''), None) + if history and history.content.get("visibility", None) == "after_join": + for ev in state.values(): + if ev.type != EventTypes.Member: + continue + try: + domain = UserID.from_string(ev.state_key).domain + except: + continue + + if domain != server_name: + continue + + if ev.membership == Membership.JOIN: + return event + else: + return prune_event(event) + + return event + + res = map(redact_disallowed, events_and_states) + + logger.info("_filter_events_for_server %r", res) + + defer.returnValue(res) + @log_function @defer.inlineCallbacks def backfill(self, dest, room_id, limit, extremities=[]): @@ -882,6 +924,8 @@ class FederationHandler(BaseHandler): limit ) + events = yield self._filter_events_for_server(origin, room_id, events) + defer.returnValue(events) @defer.inlineCallbacks From 65e69dec8b202e3f03063b18ebde55ceee69e2e7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 6 Jul 2015 09:31:40 +0100 Subject: [PATCH 090/180] Don't explode if we don't recognize one of the event_ids in the backfill request --- synapse/storage/event_federation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index ace745953..c71019d93 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -408,10 +408,12 @@ class EventFederationStore(SQLBaseStore): keyvalues={ "event_id": event_id, }, - retcol="depth" + retcol="depth", + allow_none=True, ) - queue.put((-depth, event_id)) + if depth: + queue.put((-depth, event_id)) while not queue.empty() and len(event_results) < limit: try: From fb47c3cfbe213c01b25e5605b81c998b764e2bf8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 6 Jul 2015 13:05:52 +0100 Subject: [PATCH 091/180] Rename key and values for m.room.history_visibility. Support 'invited' value --- synapse/events/utils.py | 2 +- synapse/handlers/federation.py | 34 ++++++++++++++++++++-------------- synapse/handlers/message.py | 24 ++++++++++++++++++++---- synapse/handlers/sync.py | 25 ++++++++++++++++++++----- 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 4c82780f4..7bd78343f 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -75,7 +75,7 @@ def prune_event(event): elif event_type == EventTypes.Aliases: add_fields("aliases") elif event_type == EventTypes.RoomHistoryVisibility: - add_fields("visibility") + add_fields("history_visibility") allowed_fields = { k: v diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 663d05c63..cd3867ed9 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -239,22 +239,28 @@ class FederationHandler(BaseHandler): return event history = state.get((EventTypes.RoomHistoryVisibility, ''), None) - if history and history.content.get("visibility", None) == "after_join": - for ev in state.values(): - if ev.type != EventTypes.Member: - continue - try: - domain = UserID.from_string(ev.state_key).domain - except: - continue + if history: + visibility = history.content.get("history_visibility", "shared") + if visibility in ["invited", "joined"]: + for ev in state.values(): + if ev.type != EventTypes.Member: + continue + try: + domain = UserID.from_string(ev.state_key).domain + except: + continue - if domain != server_name: - continue + if domain != server_name: + continue - if ev.membership == Membership.JOIN: - return event - else: - return prune_event(event) + memtype = ev.membership + if memtype == Membership.JOIN: + return event + elif memtype == Membership.INVITE: + if visibility == "invited": + return event + else: + return prune_event(event) return event diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 00c7dbec8..d8b117612 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -149,13 +149,29 @@ class MessageHandler(BaseHandler): if event.type == EventTypes.RoomHistoryVisibility: return True - membership = state.get((EventTypes.Member, user_id), None) - if membership and membership.membership == Membership.JOIN: + membership_ev = state.get((EventTypes.Member, user_id), None) + if membership_ev: + membership = membership_ev.membership + else: + membership = Membership.LEAVE + + if membership == Membership.JOIN: return True history = state.get((EventTypes.RoomHistoryVisibility, ''), None) - if history and history.content.get("visibility", None) == "after_join": - return False + if history: + visibility = history.content.get("history_visibility", "shared") + else: + visibility = "shared" + + if visibility == "public": + return True + elif visibility == "shared": + return True + elif visibility == "joined": + return membership == Membership.JOIN + elif visibility == "invited": + return membership == Membership.INVITE return True diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 5078c4e45..6cff6230c 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -306,16 +306,31 @@ class SyncHandler(BaseHandler): if event.type == EventTypes.RoomHistoryVisibility: return True - membership = state.get((EventTypes.Member, user_id), None) - if membership and membership.membership == Membership.JOIN: + membership_ev = state.get((EventTypes.Member, user_id), None) + if membership_ev: + membership = membership_ev.membership + else: + membership = Membership.LEAVE + + if membership == Membership.JOIN: return True history = state.get((EventTypes.RoomHistoryVisibility, ''), None) - if history and history.content.get("visibility", None) == "after_join": - return False + if history: + visibility = history.content.get("history_visibility", "shared") + else: + visibility = "shared" + + if visibility == "public": + return True + elif visibility == "shared": + return True + elif visibility == "joined": + return membership == Membership.JOIN + elif visibility == "invited": + return membership == Membership.INVITE return True - events_and_states = filter(allowed, events_and_states) defer.returnValue([ ev From 1a3255b507550c76f11251c890a43947b1f4e272 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 6 Jul 2015 13:09:16 +0100 Subject: [PATCH 092/180] Add m.room.history_visibility to newly created rooms' m.room.power_levels --- synapse/api/auth.py | 1 + synapse/handlers/room.py | 1 + 2 files changed, 2 insertions(+) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index deca747f7..1a25bf108 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -575,6 +575,7 @@ class Auth(object): levels_to_check = [ ("users_default", []), ("events_default", []), + ("state_default", []), ("ban", []), ("redact", []), ("kick", []), diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 4bd027d9b..891707df4 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -213,6 +213,7 @@ class RoomCreationHandler(BaseHandler): "events": { EventTypes.Name: 100, EventTypes.PowerLevels: 100, + EventTypes.RoomHistoryVisibility: 100, }, "events_default": 0, "state_default": 50, From a7dcbfe430f31a6e9fc6b5920ddff39eb1b1e3b4 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 6 Jul 2015 16:47:17 +0100 Subject: [PATCH 093/180] Don't 500 if a group is missing from the regex --- synapse/http/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 807ff95c6..b60e905a6 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -207,7 +207,7 @@ class JsonResource(HttpServer, resource.Resource): incoming_requests_counter.inc(request.method, servlet_classname) args = [ - urllib.unquote(u).decode("UTF-8") for u in m.groups() + urllib.unquote(u).decode("UTF-8") if u else u for u in m.groups() ] callback_return = yield callback(request, *args) From b5770f89478509b5c4c1a610753989fbe29e35e7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 6 Jul 2015 18:46:47 +0100 Subject: [PATCH 094/180] Add store for client end to end keys --- synapse/storage/__init__.py | 4 +- synapse/storage/end_to_end_keys.py | 133 ++++++++++++++++++ .../schema/delta/21/end_to_end_keys.sql | 35 +++++ 3 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/end_to_end_keys.py create mode 100644 synapse/storage/schema/delta/21/end_to_end_keys.sql diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c137f4782..e089d8167 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -37,6 +37,7 @@ from .rejections import RejectionsStore from .state import StateStore from .signatures import SignatureStore from .filtering import FilteringStore +from .end_to_end_keys import EndToEndKeyStore import fnmatch @@ -51,7 +52,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 20 +SCHEMA_VERSION = 21 dir_path = os.path.abspath(os.path.dirname(__file__)) @@ -74,6 +75,7 @@ class DataStore(RoomMemberStore, RoomStore, PushRuleStore, ApplicationServiceTransactionStore, EventsStore, + EndToEndKeyStore, ): def __init__(self, hs): diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py new file mode 100644 index 000000000..936a64669 --- /dev/null +++ b/synapse/storage/end_to_end_keys.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from _base import SQLBaseStore + + +class EndToEndKeyStore(SQLBaseStore): + def set_e2e_device_keys(self, user_id, device_id, time_now, json_bytes): + return self._simple_upsert( + table="e2e_device_keys_json", + keyvalues={ + "user_id": user_id, + "device_id": device_id, + }, + values={ + "ts_added_ms": time_now, + "key_json": json_bytes, + } + ) + + def get_e2e_device_keys(self, query_list): + """Fetch a list of device keys. + Args: + query_list(list): List of pairs of user_ids and device_ids. + Returns: + Dict mapping from user-id to dict mapping from device_id to + key json byte strings. + """ + def _get_e2e_device_keys(txn): + result = {} + for user_id, device_id in query_list: + user_result = result.setdefault(user_id, {}) + keyvalues = {"user_id": user_id} + if device_id: + keyvalues["device_id"] = device_id + rows = self._simple_select_list_txn( + txn, table="e2e_device_keys_json", + keyvalues=keyvalues, + retcols=["device_id", "key_json"] + ) + for row in rows: + user_result[row["device_id"]] = row["key_json"] + return result + return self.runInteraction("get_e2e_device_keys", _get_e2e_device_keys) + + def add_e2e_one_time_keys(self, user_id, device_id, time_now, valid_until, + key_list): + def _add_e2e_one_time_keys(txn): + for (algorithm, key_id, json_bytes) in key_list: + self._simple_upsert_txn( + txn, table="e2e_one_time_keys_json", + keyvalues={ + "user_id": user_id, + "device_id": device_id, + "algorithm": algorithm, + "key_id": key_id, + }, + values={ + "ts_added_ms": time_now, + "valid_until_ms": valid_until, + "key_json": json_bytes, + } + ) + return self.runInteraction( + "add_e2e_one_time_keys", _add_e2e_one_time_keys + ) + + def count_e2e_one_time_keys(self, user_id, device_id, time_now): + """ Count the number of one time keys the server has for a device + Returns: + Dict mapping from algorithm to number of keys for that algorithm. + """ + def _count_e2e_one_time_keys(txn): + sql = ( + "DELETE FROM e2e_one_time_keys_json" + " WHERE user_id = ? AND device_id = ? AND valid_until_ms < ?" + ) + txn.execute(sql, (user_id, device_id, time_now)) + sql = ( + "SELECT algorithm, COUNT(key_id) FROM e2e_one_time_keys_json" + " WHERE user_id = ? AND device_id = ?" + " GROUP BY algorithm" + ) + txn.execute(sql, (user_id, device_id)) + result = {} + for algorithm, key_count in txn.fetchall(): + result[algorithm] = key_count + return result + return self.runInteraction( + "count_e2e_one_time_keys", _count_e2e_one_time_keys + ) + + def take_e2e_one_time_keys(self, query_list, time_now): + """Take a list of one time keys out of the database""" + def _take_e2e_one_time_keys(txn): + sql = ( + "SELECT key_id, key_json FROM e2e_one_time_keys_json" + " WHERE user_id = ? AND device_id = ? AND algorithm = ?" + " AND valid_until_ms > ?" + " LIMIT 1" + ) + result = {} + delete = [] + for user_id, device_id, algorithm in query_list: + user_result = result.setdefault(user_id, {}) + device_result = user_result.setdefault(device_id, {}) + txn.execute(sql, (user_id, device_id, algorithm, time_now)) + for key_id, key_json in txn.fetchall(): + device_result[algorithm + ":" + key_id] = key_json + delete.append((user_id, device_id, algorithm, key_id)) + sql = ( + "DELETE FROM e2e_one_time_keys_json" + " WHERE user_id = ? AND device_id = ? AND algorithm = ?" + " AND key_id = ?" + ) + for user_id, device_id, algorithm, key_id in delete: + txn.execute(sql, (user_id, device_id, algorithm, key_id)) + return result + return self.runInteraction( + "take_e2e_one_time_keys", _take_e2e_one_time_keys + ) diff --git a/synapse/storage/schema/delta/21/end_to_end_keys.sql b/synapse/storage/schema/delta/21/end_to_end_keys.sql new file mode 100644 index 000000000..107d2e67c --- /dev/null +++ b/synapse/storage/schema/delta/21/end_to_end_keys.sql @@ -0,0 +1,35 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +CREATE TABLE IF NOT EXISTS e2e_device_keys_json ( + user_id TEXT NOT NULL, -- The user these keys are for. + device_id TEXT NOT NULL, -- Which of the user's devices these keys are for. + ts_added_ms BIGINT NOT NULL, -- When the keys were uploaded. + key_json TEXT NOT NULL, -- The keys for the device as a JSON blob. + CONSTRAINT uniqueness UNIQUE (user_id, device_id) +); + + +CREATE TABLE IF NOT EXISTS e2e_one_time_keys_json ( + user_id TEXT NOT NULL, -- The user this one-time key is for. + device_id TEXT NOT NULL, -- The device this one-time key is for. + algorithm TEXT NOT NULL, -- Which algorithm this one-time key is for. + key_id TEXT NOT NULL, -- An id for suppressing duplicate uploads. + ts_added_ms BIGINT NOT NULL, -- When this key was uploaded. + valid_until_ms BIGINT NOT NULL, -- When this key is valid until. + key_json TEXT NOT NULL, -- The key as a JSON blob. + CONSTRAINT uniqueness UNIQUE (user_id, device_id, algorithm, key_id) +); From 2ef182ee9358bce24cdef7c09ae7289925d076ef Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 6 Jul 2015 18:47:57 +0100 Subject: [PATCH 095/180] Add client API for uploading and querying keys for end to end encryption --- synapse/rest/client/v2_alpha/__init__.py | 4 +- synapse/rest/client/v2_alpha/keys.py | 287 +++++++++++++++++++++++ 2 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/client/v2_alpha/keys.py diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 7d1aff430..c3323d2a8 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -18,7 +18,8 @@ from . import ( filter, account, register, - auth + auth, + keys, ) from synapse.http.server import JsonResource @@ -38,3 +39,4 @@ class ClientV2AlphaRestResource(JsonResource): account.register_servlets(hs, client_resource) register.register_servlets(hs, client_resource) auth.register_servlets(hs, client_resource) + keys.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/v2_alpha/keys.py new file mode 100644 index 000000000..3bb4ad64f --- /dev/null +++ b/synapse/rest/client/v2_alpha/keys.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import SynapseError +from synapse.http.servlet import RestServlet +from syutil.jsonutil import encode_canonical_json + +from ._base import client_v2_pattern + +import simplejson as json +import logging + +logger = logging.getLogger(__name__) + + +class KeyUploadServlet(RestServlet): + """ + POST /keys/upload/ HTTP/1.1 + Content-Type: application/json + + { + "device_keys": { + "user_id": "", + "device_id": "", + "valid_until_ts": , + "algorithms": [ + "m.olm.curve25519-aes-sha256", + ] + "keys": { + ":": "", + }, + "signatures:" { + "/" { + ":": "" + } } }, + "one_time_keys": { + ":": "" + }, + "one_time_keys_valid_for": , + } + """ + PATTERN = client_v2_pattern("/keys/upload/(?P[^/]*)") + + def __init__(self, hs): + super(KeyUploadServlet, self).__init__() + self.store = hs.get_datastore() + self.clock = hs.get_clock() + self.auth = hs.get_auth() + + @defer.inlineCallbacks + def on_POST(self, request, device_id): + auth_user, client_info = yield self.auth.get_user_by_req(request) + user_id = auth_user.to_string() + # TODO: Check that the device_id matches that in the authentication + # or derive the device_id from the authentication instead. + try: + body = json.loads(request.content.read()) + except: + raise SynapseError(400, "Invalid key JSON") + time_now = self.clock.time_msec() + + # TODO: Validate the JSON to make sure it has the right keys. + device_keys = body.get("device_keys", None) + if device_keys: + logger.info( + "Updating device_keys for device %r for user %r at %d", + device_id, auth_user, time_now + ) + # TODO: Sign the JSON with the server key + yield self.store.set_e2e_device_keys( + user_id, device_id, time_now, + encode_canonical_json(device_keys) + ) + + one_time_keys = body.get("one_time_keys", None) + one_time_keys_valid_for = body.get("one_time_keys_valid_for", None) + if one_time_keys: + valid_until = int(one_time_keys_valid_for) + time_now + logger.info( + "Adding %d one_time_keys for device %r for user %r at %d" + " valid_until %d", + len(one_time_keys), device_id, user_id, time_now, valid_until + ) + key_list = [] + for key_id, key_json in one_time_keys.items(): + algorithm, key_id = key_id.split(":") + key_list.append(( + algorithm, key_id, encode_canonical_json(key_json) + )) + + yield self.store.add_e2e_one_time_keys( + user_id, device_id, time_now, valid_until, key_list + ) + + result = yield self.store.count_e2e_one_time_keys( + user_id, device_id, time_now + ) + defer.returnValue((200, {"one_time_key_counts": result})) + + @defer.inlineCallbacks + def on_GET(self, request, device_id): + auth_user, client_info = yield self.auth.get_user_by_req(request) + user_id = auth_user.to_string() + time_now = self.clock.time_msec() + + result = yield self.store.count_e2e_one_time_keys( + user_id, device_id, time_now + ) + defer.returnValue((200, {"one_time_key_counts": result})) + + +class KeyQueryServlet(RestServlet): + """ + GET /keys/query/ HTTP/1.1 + + GET /keys/query// HTTP/1.1 + + POST /keys/query HTTP/1.1 + Content-Type: application/json + { + "device_keys": { + "": [""] + } } + + HTTP/1.1 200 OK + { + "device_keys": { + "": { + "": { + "user_id": "", // Duplicated to be signed + "device_id": "", // Duplicated to be signed + "valid_until_ts": , + "algorithms": [ // List of supported algorithms + "m.olm.curve25519-aes-sha256", + ], + "keys": { // Must include a ed25519 signing key + ":": "", + }, + "signatures:" { + // Must be signed with device's ed25519 key + "/": { + ":": "" + } + // Must be signed by this server. + "": { + ":": "" + } } } } } } + """ + + PATTERN = client_v2_pattern( + "/keys/query(?:" + "/(?P[^/]*)(?:" + "/(?P[^/]*)" + ")?" + ")?" + ) + + def __init__(self, hs): + super(KeyQueryServlet, self).__init__() + self.store = hs.get_datastore() + self.auth = hs.get_auth() + + @defer.inlineCallbacks + def on_POST(self, request, user_id, device_id): + logger.debug("onPOST") + yield self.auth.get_user_by_req(request) + try: + body = json.loads(request.content.read()) + except: + raise SynapseError(400, "Invalid key JSON") + query = [] + for user_id, device_ids in body.get("device_keys", {}).items(): + if not device_ids: + query.append((user_id, None)) + else: + for device_id in device_ids: + query.append((user_id, device_id)) + results = yield self.store.get_e2e_device_keys([(user_id, device_id)]) + defer.returnValue(self.json_result(request, results)) + + @defer.inlineCallbacks + def on_GET(self, request, user_id, device_id): + auth_user, client_info = yield self.auth.get_user_by_req(request) + auth_user_id = auth_user.to_string() + if not user_id: + user_id = auth_user_id + if not device_id: + device_id = None + # Returns a map of user_id->device_id->json_bytes. + results = yield self.store.get_e2e_device_keys([(user_id, device_id)]) + defer.returnValue(self.json_result(request, results)) + + def json_result(self, request, results): + json_result = {} + for user_id, device_keys in results.items(): + for device_id, json_bytes in device_keys.items(): + json_result.setdefault(user_id, {})[device_id] = json.loads( + json_bytes + ) + return (200, {"device_keys": json_result}) + + +class OneTimeKeyServlet(RestServlet): + """ + GET /keys/take/// HTTP/1.1 + + POST /keys/take HTTP/1.1 + { + "one_time_keys": { + "": { + "": "" + } } } + + HTTP/1.1 200 OK + { + "one_time_keys": { + "": { + "": { + ":": "" + } } } } + + """ + PATTERN = client_v2_pattern( + "/keys/take(?:/?|(?:/" + "(?P[^/]*)/(?P[^/]*)/(?P[^/]*)" + ")?)" + ) + + def __init__(self, hs): + super(OneTimeKeyServlet, self).__init__() + self.store = hs.get_datastore() + self.auth = hs.get_auth() + self.clock = hs.get_clock() + + @defer.inlineCallbacks + def on_GET(self, request, user_id, device_id, algorithm): + yield self.auth.get_user_by_req(request) + time_now = self.clock.time_msec() + results = yield self.store.take_e2e_one_time_keys( + [(user_id, device_id, algorithm)], time_now + ) + defer.returnValue(self.json_result(request, results)) + + @defer.inlineCallbacks + def on_POST(self, request, user_id, device_id, algorithm): + yield self.auth.get_user_by_req(request) + try: + body = json.loads(request.content.read()) + except: + raise SynapseError(400, "Invalid key JSON") + query = [] + for user_id, device_keys in body.get("one_time_keys", {}).items(): + for device_id, algorithm in device_keys.items(): + query.append((user_id, device_id, algorithm)) + time_now = self.clock.time_msec() + results = yield self.store.take_e2e_one_time_keys(query, time_now) + defer.returnValue(self.json_result(request, results)) + + def json_result(self, request, results): + json_result = {} + for user_id, device_keys in results.items(): + for device_id, keys in device_keys.items(): + for key_id, json_bytes in keys.items(): + json_result.setdefault(user_id, {})[device_id] = { + key_id: json.loads(json_bytes) + } + return (200, {"one_time_keys": json_result}) + + +def register_servlets(hs, http_server): + KeyUploadServlet(hs).register(http_server) + KeyQueryServlet(hs).register(http_server) + OneTimeKeyServlet(hs).register(http_server) From e8b2f6f8a13c1b419911a54f349d05fe97b5ace0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Jul 2015 10:55:12 +0100 Subject: [PATCH 096/180] Add a ReceiptServlet --- synapse/rest/client/v2_alpha/__init__.py | 4 +- synapse/rest/client/v2_alpha/receipts.py | 56 ++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/client/v2_alpha/receipts.py diff --git a/synapse/rest/client/v2_alpha/__init__.py b/synapse/rest/client/v2_alpha/__init__.py index 7d1aff430..231e1dd97 100644 --- a/synapse/rest/client/v2_alpha/__init__.py +++ b/synapse/rest/client/v2_alpha/__init__.py @@ -18,7 +18,8 @@ from . import ( filter, account, register, - auth + auth, + receipts, ) from synapse.http.server import JsonResource @@ -38,3 +39,4 @@ class ClientV2AlphaRestResource(JsonResource): account.register_servlets(hs, client_resource) register.register_servlets(hs, client_resource) auth.register_servlets(hs, client_resource) + receipts.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/receipts.py b/synapse/rest/client/v2_alpha/receipts.py new file mode 100644 index 000000000..829427b7b --- /dev/null +++ b/synapse/rest/client/v2_alpha/receipts.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.http.servlet import RestServlet +from ._base import client_v2_pattern + +import logging + + +logger = logging.getLogger(__name__) + + +class ReceiptRestServlet(RestServlet): + PATTERN = client_v2_pattern( + "/rooms/(?P[^/]*)" + "/receipt/(?P[^/]*)" + "/(?P[^/])*" + ) + + def __init__(self, hs): + super(ReceiptRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + self.receipts_handler = hs.get_handlers().receipts_handler + + @defer.inlineCallbacks + def on_POST(self, request, room_id, receipt_type, event_id): + user, client = yield self.auth.get_user_by_req(request) + + # TODO: STUFF + yield self.receipts_handler.received_client_receipt( + room_id, + receipt_type, + user_id=user.to_string(), + event_id=event_id + ) + + defer.returnValue((200, {})) + + +def register_servlets(hs, http_server): + ReceiptRestServlet(hs).register(http_server) From 716e42693354553ee2878e3a8df6811226e91130 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Jul 2015 10:55:31 +0100 Subject: [PATCH 097/180] Fix various typos --- synapse/handlers/__init__.py | 2 ++ synapse/handlers/receipts.py | 6 ++++-- synapse/storage/__init__.py | 3 +++ synapse/storage/receipts.py | 13 +++++++------ 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index 685792dbd..dc5b6ef79 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -32,6 +32,7 @@ from .appservice import ApplicationServicesHandler from .sync import SyncHandler from .auth import AuthHandler from .identity import IdentityHandler +from .receipts import ReceiptsHandler class Handlers(object): @@ -57,6 +58,7 @@ class Handlers(object): self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) self.admin_handler = AdminHandler(hs) + self.receipts_handler = ReceiptsHandler(hs) asapi = ApplicationServiceApi(hs) self.appservice_handler = ApplicationServicesHandler( hs, asapi, AppServiceScheduler( diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index f0d12d35f..fc2f38c1c 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -31,6 +31,8 @@ class ReceiptsHandler(BaseHandler): def __init__(self, hs): super(ReceiptsHandler, self).__init__(hs) + self.hs = hs + self.federation = hs.get_replication_layer() self.federation.register_edu_handler( "m.receipt", self._received_remote_receipt ) @@ -89,13 +91,13 @@ class ReceiptsHandler(BaseHandler): with PreserveLoggingContext(): self.notifier.on_new_event( - "recei[t_key", self._latest_serial, rooms=[room_id] + "receipt_key", self._latest_serial, rooms=[room_id] ) localusers = set() remotedomains = set() - rm_handler = self.homeserver.get_handlers().room_member_handler + rm_handler = self.hs.get_handlers().room_member_handler yield rm_handler.fetch_room_distributions_into( room_id, localusers=localusers, remotedomains=remotedomains ) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 275598add..2bc88a795 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -38,6 +38,8 @@ from .state import StateStore from .signatures import SignatureStore from .filtering import FilteringStore +from .receipts import ReceiptsStore + import fnmatch import imp @@ -74,6 +76,7 @@ class DataStore(RoomMemberStore, RoomStore, PushRuleStore, ApplicationServiceTransactionStore, EventsStore, + ReceiptsStore, ): def __init__(self, hs): diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 0168e74a0..15c11fd41 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -18,7 +18,7 @@ from ._base import SQLBaseStore, cached from twisted.internet import defer -class ReceiptStore(SQLBaseStore): +class ReceiptsStore(SQLBaseStore): @cached @defer.inlineCallbacks @@ -77,6 +77,7 @@ class ReceiptStore(SQLBaseStore): txn, table="receipts_linearized", values={ + "stream_id": stream_id, "room_id": room_id, "receipt_type": receipt_type, "user_id": user_id, @@ -109,16 +110,16 @@ class ReceiptStore(SQLBaseStore): return None linearized_event_id = yield self.runInteraction( - graph_to_linear, desc="insert_receipt_conv" + "insert_receipt_conv", graph_to_linear ) - stream_id_manager = yield self._stream_id_gen.get_next(self) - with stream_id_manager() as stream_id: + stream_id_manager = yield self._receipts_id_gen.get_next(self) + with stream_id_manager as stream_id: yield self.runInteraction( + "insert_linearized_receipt", self.insert_linearized_receipt_txn, room_id, receipt_type, user_id, linearized_event_id, stream_id=stream_id, - desc="insert_linearized_receipt" ) yield self.insert_graph_receipt( @@ -131,9 +132,9 @@ class ReceiptStore(SQLBaseStore): def insert_graph_receipt(self, room_id, receipt_type, user_id, event_ids): return self.runInteraction( + "insert_graph_receipt", self.insert_graph_receipt_txn, room_id, receipt_type, user_id, event_ids, - desc="insert_graph_receipt" ) def insert_graph_receipt_txn(self, txn, room_id, receipt_type, From ca041d55267740214a2cfab95c44ee6f70cc6d0d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Jul 2015 15:25:30 +0100 Subject: [PATCH 098/180] Wire together receipts and the notifer/federation --- synapse/handlers/receipts.py | 81 ++++++++++++++++++------ synapse/rest/client/v2_alpha/receipts.py | 3 +- synapse/storage/receipts.py | 69 +++++++++++++++++--- synapse/streams/events.py | 6 +- 4 files changed, 126 insertions(+), 33 deletions(-) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index fc2f38c1c..94f081005 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -37,7 +37,8 @@ class ReceiptsHandler(BaseHandler): "m.receipt", self._received_remote_receipt ) - self._latest_serial = 0 + # self._earliest_cached_serial = 0 + # self._rooms_to_latest_serial = {} @defer.inlineCallbacks def received_client_receipt(self, room_id, receipt_type, user_id, @@ -53,8 +54,10 @@ class ReceiptsHandler(BaseHandler): "event_ids": [event_id], } - yield self._handle_new_receipts([receipt]) - self._push_remotes([receipt]) + is_new = yield self._handle_new_receipts([receipt]) + + if is_new: + self._push_remotes([receipt]) @defer.inlineCallbacks def _received_remote_receipt(self, origin, content): @@ -81,33 +84,24 @@ class ReceiptsHandler(BaseHandler): user_id = receipt["user_id"] event_ids = receipt["event_ids"] - stream_id, max_persisted_id = yield self.store.insert_receipt( + res = yield self.store.insert_receipt( room_id, receipt_type, user_id, event_ids, ) - # TODO: Use max_persisted_id + if not res: + # res will be None if this read receipt is 'old' + defer.returnValue(False) - self._latest_serial = max(self._latest_serial, stream_id) + stream_id, max_persisted_id = res with PreserveLoggingContext(): self.notifier.on_new_event( - "receipt_key", self._latest_serial, rooms=[room_id] + "receipt_key", max_persisted_id, rooms=[room_id] ) - localusers = set() - remotedomains = set() - - rm_handler = self.hs.get_handlers().room_member_handler - yield rm_handler.fetch_room_distributions_into( - room_id, localusers=localusers, remotedomains=remotedomains - ) - - receipt["remotedomains"] = remotedomains - - self.notifier.on_new_event( - "receipt_key", self._latest_room_serial, rooms=[room_id] - ) + defer.returnValue(True) + @defer.inlineCallbacks def _push_remotes(self, receipts): # TODO: Some of this stuff should be coallesced. for receipt in receipts: @@ -115,7 +109,15 @@ class ReceiptsHandler(BaseHandler): receipt_type = receipt["receipt_type"] user_id = receipt["user_id"] event_ids = receipt["event_ids"] - remotedomains = receipt["remotedomains"] + + remotedomains = set() + + rm_handler = self.hs.get_handlers().room_member_handler + yield rm_handler.fetch_room_distributions_into( + room_id, localusers=None, remotedomains=remotedomains + ) + + logger.debug("Sending receipt to: %r", remotedomains) for domain in remotedomains: self.federation.send_edu( @@ -130,3 +132,40 @@ class ReceiptsHandler(BaseHandler): }, }, ) + + +class ReceiptEventSource(object): + def __init__(self, hs): + self.store = hs.get_datastore() + + @defer.inlineCallbacks + def get_new_events_for_user(self, user, from_key, limit): + from_key = int(from_key) + to_key = yield self.get_current_key() + + rooms = yield self.store.get_rooms_for_user(user.to_string()) + rooms = [room.room_id for room in rooms] + content = {} + for room_id in rooms: + result = yield self.store.get_linearized_receipts_for_room( + room_id, from_key, to_key + ) + if result: + content[room_id] = result + + if not content: + defer.returnValue(([], to_key)) + + event = { + "type": "m.receipt", + "content": content, + } + + defer.returnValue(([event], to_key)) + + def get_current_key(self, direction='f'): + return self.store.get_max_receipt_stream_id() + + @defer.inlineCallbacks + def get_pagination_rows(self, user, config, key): + defer.returnValue(([{}], 0)) diff --git a/synapse/rest/client/v2_alpha/receipts.py b/synapse/rest/client/v2_alpha/receipts.py index 829427b7b..40406e2ed 100644 --- a/synapse/rest/client/v2_alpha/receipts.py +++ b/synapse/rest/client/v2_alpha/receipts.py @@ -28,7 +28,7 @@ class ReceiptRestServlet(RestServlet): PATTERN = client_v2_pattern( "/rooms/(?P[^/]*)" "/receipt/(?P[^/]*)" - "/(?P[^/])*" + "/(?P[^/]*)$" ) def __init__(self, hs): @@ -41,7 +41,6 @@ class ReceiptRestServlet(RestServlet): def on_POST(self, request, room_id, receipt_type, event_id): user, client = yield self.auth.get_user_by_req(request) - # TODO: STUFF yield self.receipts_handler.received_client_receipt( room_id, receipt_type, diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 15c11fd41..5a02c8025 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -17,17 +17,33 @@ from ._base import SQLBaseStore, cached from twisted.internet import defer +import logging + + +logger = logging.getLogger(__name__) + class ReceiptsStore(SQLBaseStore): - @cached @defer.inlineCallbacks - def get_linearized_receipts_for_room(self, room_id): - rows = yield self._simple_select_list( - table="receipts_linearized", - keyvalues={"room_id": room_id}, - retcols=["receipt_type", "user_id", "event_id"], - desc="get_linearized_receipts_for_room", + def get_linearized_receipts_for_room(self, room_id, from_key, to_key): + def f(txn): + sql = ( + "SELECT * FROM receipts_linearized WHERE" + " room_id = ? AND stream_id > ? AND stream_id <= ?" + ) + + txn.execute( + sql, + (room_id, from_key, to_key) + ) + + rows = self.cursor_to_dict(txn) + + return rows + + rows = yield self.runInteraction( + "get_linearized_receipts_for_room", f ) result = {} @@ -40,6 +56,9 @@ class ReceiptsStore(SQLBaseStore): defer.returnValue(result) + def get_max_receipt_stream_id(self): + return self._receipts_id_gen.get_max_token(self) + @cached @defer.inlineCallbacks def get_graph_receipts_for_room(self, room_id): @@ -62,11 +81,38 @@ class ReceiptsStore(SQLBaseStore): def insert_linearized_receipt_txn(self, txn, room_id, receipt_type, user_id, event_id, stream_id): + + # We don't want to clobber receipts for more recent events, so we + # have to compare orderings of existing receipts + sql = ( + "SELECT topological_ordering, stream_ordering, event_id FROM events" + " INNER JOIN receipts_linearized as r USING (event_id, room_id)" + " WHERE r.room_id = ? AND r.receipt_type = ? AND r.user_id = ?" + ) + + txn.execute(sql, (room_id, receipt_type, user_id)) + results = txn.fetchall() + + if results: + res = self._simple_select_one_txn( + txn, + table="events", + retcols=["topological_ordering", "stream_ordering"], + keyvalues={"event_id": event_id}, + ) + topological_ordering = int(res["topological_ordering"]) + stream_ordering = int(res["stream_ordering"]) + + for to, so, _ in results: + if int(to) > topological_ordering: + return False + elif int(to) == topological_ordering and int(so) >= stream_ordering: + return False + self._simple_delete_txn( txn, table="receipts_linearized", keyvalues={ - "stream_id": stream_id, "room_id": room_id, "receipt_type": receipt_type, "user_id": user_id, @@ -85,6 +131,8 @@ class ReceiptsStore(SQLBaseStore): } ) + return True + @defer.inlineCallbacks def insert_receipt(self, room_id, receipt_type, user_id, event_ids): if not event_ids: @@ -115,13 +163,16 @@ class ReceiptsStore(SQLBaseStore): stream_id_manager = yield self._receipts_id_gen.get_next(self) with stream_id_manager as stream_id: - yield self.runInteraction( + have_persisted = yield self.runInteraction( "insert_linearized_receipt", self.insert_linearized_receipt_txn, room_id, receipt_type, user_id, linearized_event_id, stream_id=stream_id, ) + if not have_persisted: + defer.returnValue(None) + yield self.insert_graph_receipt( room_id, receipt_type, user_id, event_ids ) diff --git a/synapse/streams/events.py b/synapse/streams/events.py index 0a1a3a3d0..aaa3609aa 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -20,6 +20,7 @@ from synapse.types import StreamToken from synapse.handlers.presence import PresenceEventSource from synapse.handlers.room import RoomEventSource from synapse.handlers.typing import TypingNotificationEventSource +from synapse.handlers.receipts import ReceiptEventSource class NullSource(object): @@ -43,6 +44,7 @@ class EventSources(object): "room": RoomEventSource, "presence": PresenceEventSource, "typing": TypingNotificationEventSource, + "receipt": ReceiptEventSource, } def __init__(self, hs): @@ -63,7 +65,9 @@ class EventSources(object): typing_key=( yield self.sources["typing"].get_current_key() ), - receipt_key="0", + receipt_key=( + yield self.sources["receipt"].get_current_key() + ), ) defer.returnValue(token) From f0dd6d4cbddd71f19b4bb20754e302eb48dd21b3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 7 Jul 2015 16:18:36 +0100 Subject: [PATCH 099/180] Fix test. --- tests/rest/client/v1/test_events.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py index 445272e32..ac3b0b58a 100644 --- a/tests/rest/client/v1/test_events.py +++ b/tests/rest/client/v1/test_events.py @@ -183,7 +183,17 @@ class EventStreamPermissionsTestCase(RestTestCase): ) self.assertEquals(200, code, msg=str(response)) - self.assertEquals(0, len(response["chunk"])) + # We may get a presence event for ourselves down + self.assertEquals( + 0, + len([ + c for c in response["chunk"] + if not ( + c.get("type") == "m.presence" + and c["content"].get("user_id") == self.user_id + ) + ]) + ) # joined room (expect all content for room) yield self.join(room=room_id, user=self.user_id, tok=self.token) From 87311d1b8cc648400dfce5db8a7fed46abbeb963 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Jul 2015 10:54:01 +0100 Subject: [PATCH 100/180] Hook up receipts to v1 initialSync --- synapse/handlers/message.py | 16 ++++++++++--- synapse/handlers/receipts.py | 45 +++++++++++++++++++++++++++++++++++- synapse/storage/receipts.py | 27 +++++++++++++++------- 3 files changed, 76 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index e324662f1..7c1d6b548 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -278,6 +278,11 @@ class MessageHandler(BaseHandler): user, pagination_config.get_source_config("presence"), None ) + receipt_stream = self.hs.get_event_sources().sources["receipt"] + receipt, _ = yield receipt_stream.get_pagination_rows( + user, pagination_config.get_source_config("receipt"), None + ) + public_room_ids = yield self.store.get_public_room_ids() limit = pagin_config.limit @@ -344,7 +349,8 @@ class MessageHandler(BaseHandler): ret = { "rooms": rooms_ret, "presence": presence, - "end": now_token.to_string() + "receipts": receipt, + "end": now_token.to_string(), } defer.returnValue(ret) @@ -405,9 +411,12 @@ class MessageHandler(BaseHandler): defer.returnValue([p for success, p in presence_defs if success]) - presence, (messages, token) = yield defer.gatherResults( + receipts_handler = self.hs.get_handlers().receipts_handler + + presence, receipts, (messages, token) = yield defer.gatherResults( [ get_presence(), + receipts_handler.get_receipts_for_room(room_id, now_token.receipt_key), self.store.get_recent_events_for_room( room_id, limit=limit, @@ -431,5 +440,6 @@ class MessageHandler(BaseHandler): "end": end_token.to_string(), }, "state": state, - "presence": presence + "presence": presence, + "receipts": receipts, }) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 94f081005..f6cde30e6 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -133,6 +133,24 @@ class ReceiptsHandler(BaseHandler): }, ) + @defer.inlineCallbacks + def get_receipts_for_room(self, room_id, to_key): + result = yield self.store.get_linearized_receipts_for_room( + room_id, None, to_key + ) + + if not result: + defer.returnValue([]) + + event = { + "type": "m.receipt", + "content": { + room_id: result, + }, + } + + defer.returnValue([event]) + class ReceiptEventSource(object): def __init__(self, hs): @@ -168,4 +186,29 @@ class ReceiptEventSource(object): @defer.inlineCallbacks def get_pagination_rows(self, user, config, key): - defer.returnValue(([{}], 0)) + to_key = int(config.from_key) + + if config.to_key: + from_key = int(config.to_key) + else: + from_key = None + + rooms = yield self.store.get_rooms_for_user(user.to_string()) + rooms = [room.room_id for room in rooms] + content = {} + for room_id in rooms: + result = yield self.store.get_linearized_receipts_for_room( + room_id, from_key, to_key + ) + if result: + content[room_id] = result + + if not content: + defer.returnValue(([], to_key)) + + event = { + "type": "m.receipt", + "content": content, + } + + defer.returnValue(([event], to_key)) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 5a02c8025..07f8edaac 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -28,15 +28,26 @@ class ReceiptsStore(SQLBaseStore): @defer.inlineCallbacks def get_linearized_receipts_for_room(self, room_id, from_key, to_key): def f(txn): - sql = ( - "SELECT * FROM receipts_linearized WHERE" - " room_id = ? AND stream_id > ? AND stream_id <= ?" - ) + if from_key: + sql = ( + "SELECT * FROM receipts_linearized WHERE" + " room_id = ? AND stream_id > ? AND stream_id <= ?" + ) - txn.execute( - sql, - (room_id, from_key, to_key) - ) + txn.execute( + sql, + (room_id, from_key, to_key) + ) + else: + sql = ( + "SELECT * FROM receipts_linearized WHERE" + " room_id = ? AND stream_id <= ?" + ) + + txn.execute( + sql, + (room_id, to_key) + ) rows = self.cursor_to_dict(txn) From 81682d0f820a6209535267a45ee28b8f66ff7794 Mon Sep 17 00:00:00 2001 From: Muthu Subramanian Date: Tue, 7 Jul 2015 17:40:30 +0530 Subject: [PATCH 101/180] Integrate SAML2 basic authentication - uses pysaml2 --- synapse/config/homeserver.py | 6 ++-- synapse/config/saml2.py | 27 ++++++++++++++ synapse/handlers/register.py | 30 ++++++++++++++++ synapse/python_dependencies.py | 1 + synapse/rest/client/v1/login.py | 62 ++++++++++++++++++++++++++++++++- 5 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 synapse/config/saml2.py diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index fe0ccb6eb..5c655c537 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -25,12 +25,12 @@ from .registration import RegistrationConfig from .metrics import MetricsConfig from .appservice import AppServiceConfig from .key import KeyConfig - +from .saml2 import SAML2Config class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, - VoipConfig, RegistrationConfig, - MetricsConfig, AppServiceConfig, KeyConfig,): + VoipConfig, RegistrationConfig, MetricsConfig, + AppServiceConfig, KeyConfig, SAML2Config, ): pass diff --git a/synapse/config/saml2.py b/synapse/config/saml2.py new file mode 100644 index 000000000..4f3a724e2 --- /dev/null +++ b/synapse/config/saml2.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 Ericsson +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + +class SAML2Config(Config): + def read_config(self, config): + self.saml2_config = config["saml2_config"] + + def default_config(self, config_dir_path, server_name): + return """ + saml2_config: + config_path: "%s/sp_conf.py" + idp_redirect_url: "http://%s/idp" + """%(config_dir_path, server_name) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 7b68585a1..4c6c5e297 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -192,6 +192,36 @@ class RegistrationHandler(BaseHandler): else: logger.info("Valid captcha entered from %s", ip) + @defer.inlineCallbacks + def register_saml2(self, localpart): + """ + Registers email_id as SAML2 Based Auth. + """ + if urllib.quote(localpart) != localpart: + raise SynapseError( + 400, + "User ID must only contain characters which do not" + " require URL encoding." + ) + user = UserID(localpart, self.hs.hostname) + user_id = user.to_string() + + yield self.check_user_id_is_valid(user_id) + token = self._generate_token(user_id) + try: + yield self.store.register( + user_id=user_id, + token=token, + password_hash=None + ) + yield self.distributor.fire("registered_user", user) + except Exception, e: + yield self.store.add_access_token_to_user(user_id, token) + # Ignore Registration errors + logger.exception(e) + defer.returnValue((user_id, token)) + + @defer.inlineCallbacks def register_email(self, threepidCreds): """ diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index f9e59dd91..17587170c 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -31,6 +31,7 @@ REQUIREMENTS = { "pillow": ["PIL"], "pydenticon": ["pydenticon"], "ujson": ["ujson"], + "pysaml2": ["saml2"], } CONDITIONAL_REQUIREMENTS = { "web_client": { diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index b2257b749..dc7615c6f 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -20,14 +20,32 @@ from synapse.types import UserID from base import ClientV1RestServlet, client_path_pattern import simplejson as json +import cgi +import urllib + +import logging +from saml2 import BINDING_HTTP_REDIRECT +from saml2 import BINDING_HTTP_POST +from saml2.metadata import create_metadata_string +from saml2 import config +from saml2.client import Saml2Client +from saml2.httputil import ServiceError +from saml2.samlp import Extensions +from saml2.extension.pefim import SPCertEnc +from saml2.s_utils import rndstr class LoginRestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/login$") PASS_TYPE = "m.login.password" + SAML2_TYPE = "m.login.saml2" + + def __init__(self, hs): + super(LoginRestServlet, self).__init__(hs) + self.idp_redirect_url = hs.config.saml2_config['idp_redirect_url'] def on_GET(self, request): - return (200, {"flows": [{"type": LoginRestServlet.PASS_TYPE}]}) + return (200, {"flows": [{"type": LoginRestServlet.PASS_TYPE}, {"type": LoginRestServlet.SAML2_TYPE}]}) def on_OPTIONS(self, request): return (200, {}) @@ -39,6 +57,14 @@ class LoginRestServlet(ClientV1RestServlet): if login_submission["type"] == LoginRestServlet.PASS_TYPE: result = yield self.do_password_login(login_submission) defer.returnValue(result) + elif login_submission["type"] == LoginRestServlet.SAML2_TYPE: + relay_state = "" + if "relay_state" in login_submission: + relay_state = "&RelayState="+urllib.quote(login_submission["relay_state"]) + result = { + "uri": "%s%s"%(self.idp_redirect_url, relay_state) + } + defer.returnValue((200, result)) else: raise SynapseError(400, "Bad login type.") except KeyError: @@ -93,6 +119,39 @@ class PasswordResetRestServlet(ClientV1RestServlet): "Missing keys. Requires 'email' and 'user_id'." ) +class SAML2RestServlet(ClientV1RestServlet): + PATTERN = client_path_pattern("/login/saml2") + + def __init__(self, hs): + super(SAML2RestServlet, self).__init__(hs) + self.sp_config = hs.config.saml2_config['config_path'] + + @defer.inlineCallbacks + def on_POST(self, request): + saml2_auth = None + try: + conf = config.SPConfig() + conf.load_file(self.sp_config) + SP = Saml2Client(conf) + saml2_auth = SP.parse_authn_request_response(request.args['SAMLResponse'][0], BINDING_HTTP_POST) + except Exception, e: # Not authenticated + logger = logging.getLogger(__name__) + logger.exception(e) + if saml2_auth and saml2_auth.status_ok() and not saml2_auth.not_signed: + username = saml2_auth.name_id.text + handler = self.handlers.registration_handler + (user_id, token) = yield handler.register_saml2(username) + # Forward to the RelayState callback along with ava + if 'RelayState' in request.args: + request.redirect(urllib.unquote(request.args['RelayState'][0])+'?status=authenticated&access_token='+token+'&user_id='+user_id+'&ava='+urllib.quote(json.dumps(saml2_auth.ava))) + request.finish() + defer.returnValue(None) + defer.returnValue((200, {"status":"authenticated", "user_id": user_id, "token": token, "ava":saml2_auth.ava})) + elif 'RelayState' in request.args: + request.redirect(urllib.unquote(request.args['RelayState'][0])+'?status=not_authenticated') + request.finish() + defer.returnValue(None) + defer.returnValue((200, {"status":"not_authenticated"})) def _parse_json(request): try: @@ -106,4 +165,5 @@ def _parse_json(request): def register_servlets(hs, http_server): LoginRestServlet(hs).register(http_server) + SAML2RestServlet(hs).register(http_server) # TODO PasswordResetRestServlet(hs).register(http_server) From 77c5db5977c3fb61d9d2906c6692ee502d477e18 Mon Sep 17 00:00:00 2001 From: Muthu Subramanian Date: Wed, 8 Jul 2015 16:05:20 +0530 Subject: [PATCH 102/180] code beautify --- synapse/rest/client/v1/login.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index dc7615c6f..b4c74c4c2 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -45,7 +45,8 @@ class LoginRestServlet(ClientV1RestServlet): self.idp_redirect_url = hs.config.saml2_config['idp_redirect_url'] def on_GET(self, request): - return (200, {"flows": [{"type": LoginRestServlet.PASS_TYPE}, {"type": LoginRestServlet.SAML2_TYPE}]}) + return (200, {"flows": [{"type": LoginRestServlet.PASS_TYPE}, + {"type": LoginRestServlet.SAML2_TYPE}]}) def on_OPTIONS(self, request): return (200, {}) @@ -60,9 +61,10 @@ class LoginRestServlet(ClientV1RestServlet): elif login_submission["type"] == LoginRestServlet.SAML2_TYPE: relay_state = "" if "relay_state" in login_submission: - relay_state = "&RelayState="+urllib.quote(login_submission["relay_state"]) + relay_state = "&RelayState="+urllib.quote( + login_submission["relay_state"]) result = { - "uri": "%s%s"%(self.idp_redirect_url, relay_state) + "uri": "%s%s" % (self.idp_redirect_url, relay_state) } defer.returnValue((200, result)) else: @@ -119,6 +121,7 @@ class PasswordResetRestServlet(ClientV1RestServlet): "Missing keys. Requires 'email' and 'user_id'." ) + class SAML2RestServlet(ClientV1RestServlet): PATTERN = client_path_pattern("/login/saml2") @@ -133,25 +136,35 @@ class SAML2RestServlet(ClientV1RestServlet): conf = config.SPConfig() conf.load_file(self.sp_config) SP = Saml2Client(conf) - saml2_auth = SP.parse_authn_request_response(request.args['SAMLResponse'][0], BINDING_HTTP_POST) - except Exception, e: # Not authenticated + saml2_auth = SP.parse_authn_request_response( + request.args['SAMLResponse'][0], BINDING_HTTP_POST) + except Exception, e: # Not authenticated logger = logging.getLogger(__name__) logger.exception(e) - if saml2_auth and saml2_auth.status_ok() and not saml2_auth.not_signed: + if saml2_auth and saml2_auth.status_ok() and not saml2_auth.not_signed: username = saml2_auth.name_id.text handler = self.handlers.registration_handler (user_id, token) = yield handler.register_saml2(username) # Forward to the RelayState callback along with ava if 'RelayState' in request.args: - request.redirect(urllib.unquote(request.args['RelayState'][0])+'?status=authenticated&access_token='+token+'&user_id='+user_id+'&ava='+urllib.quote(json.dumps(saml2_auth.ava))) + request.redirect(urllib.unquote( + request.args['RelayState'][0]) + + '?status=authenticated&access_token=' + + token + '&user_id=' + user_id + '&ava=' + + urllib.quote(json.dumps(saml2_auth.ava))) request.finish() defer.returnValue(None) - defer.returnValue((200, {"status":"authenticated", "user_id": user_id, "token": token, "ava":saml2_auth.ava})) + defer.returnValue((200, {"status": "authenticated", + "user_id": user_id, "token": token, + "ava": saml2_auth.ava})) elif 'RelayState' in request.args: - request.redirect(urllib.unquote(request.args['RelayState'][0])+'?status=not_authenticated') + request.redirect(urllib.unquote( + request.args['RelayState'][0]) + + '?status=not_authenticated') request.finish() defer.returnValue(None) - defer.returnValue((200, {"status":"not_authenticated"})) + defer.returnValue((200, {"status": "not_authenticated"})) + def _parse_json(request): try: From f53bae0c1948a8c0a229e0b20f237f7ff4b1d84c Mon Sep 17 00:00:00 2001 From: Muthu Subramanian Date: Wed, 8 Jul 2015 16:05:46 +0530 Subject: [PATCH 103/180] code beautify --- synapse/config/homeserver.py | 1 + synapse/config/saml2.py | 3 ++- synapse/handlers/register.py | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 5c655c537..d77f04540 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -27,6 +27,7 @@ from .appservice import AppServiceConfig from .key import KeyConfig from .saml2 import SAML2Config + class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, VoipConfig, RegistrationConfig, MetricsConfig, diff --git a/synapse/config/saml2.py b/synapse/config/saml2.py index 4f3a724e2..d18d076a8 100644 --- a/synapse/config/saml2.py +++ b/synapse/config/saml2.py @@ -15,6 +15,7 @@ from ._base import Config + class SAML2Config(Config): def read_config(self, config): self.saml2_config = config["saml2_config"] @@ -24,4 +25,4 @@ class SAML2Config(Config): saml2_config: config_path: "%s/sp_conf.py" idp_redirect_url: "http://%s/idp" - """%(config_dir_path, server_name) + """ % (config_dir_path, server_name) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 4c6c5e297..a1288b425 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -220,7 +220,6 @@ class RegistrationHandler(BaseHandler): # Ignore Registration errors logger.exception(e) defer.returnValue((user_id, token)) - @defer.inlineCallbacks def register_email(self, threepidCreds): From d85ce8d89bc1fb6ff6f277fb424dddca7ab2f47e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Jul 2015 11:36:05 +0100 Subject: [PATCH 104/180] Split receipt events up into one per room --- synapse/handlers/receipts.py | 49 +++++++++++++++--------------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index f6cde30e6..b7567b9ea 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -144,9 +144,8 @@ class ReceiptsHandler(BaseHandler): event = { "type": "m.receipt", - "content": { - room_id: result, - }, + "room_id": room_id, + "content": result, } defer.returnValue([event]) @@ -163,23 +162,19 @@ class ReceiptEventSource(object): rooms = yield self.store.get_rooms_for_user(user.to_string()) rooms = [room.room_id for room in rooms] - content = {} + events = [] for room_id in rooms: - result = yield self.store.get_linearized_receipts_for_room( + content = yield self.store.get_linearized_receipts_for_room( room_id, from_key, to_key ) - if result: - content[room_id] = result + if content: + events.append({ + "type": "m.receipt", + "room_id": room_id, + "content": content, + }) - if not content: - defer.returnValue(([], to_key)) - - event = { - "type": "m.receipt", - "content": content, - } - - defer.returnValue(([event], to_key)) + defer.returnValue((events, to_key)) def get_current_key(self, direction='f'): return self.store.get_max_receipt_stream_id() @@ -195,20 +190,16 @@ class ReceiptEventSource(object): rooms = yield self.store.get_rooms_for_user(user.to_string()) rooms = [room.room_id for room in rooms] - content = {} + events = [] for room_id in rooms: - result = yield self.store.get_linearized_receipts_for_room( + content = yield self.store.get_linearized_receipts_for_room( room_id, from_key, to_key ) - if result: - content[room_id] = result + if content: + events.append({ + "type": "m.receipt", + "room_id": room_id, + "content": content, + }) - if not content: - defer.returnValue(([], to_key)) - - event = { - "type": "m.receipt", - "content": content, - } - - defer.returnValue(([event], to_key)) + defer.returnValue((events, to_key)) From af812b68ddbd1a69a8c98c463248d000633b075f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Jul 2015 15:35:00 +0100 Subject: [PATCH 105/180] Add a cache to fetching of receipt streams --- synapse/handlers/receipts.py | 31 +++--------- synapse/storage/receipts.py | 92 ++++++++++++++++++++++++++++++++++-- 2 files changed, 96 insertions(+), 27 deletions(-) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index b7567b9ea..053ed8480 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -37,8 +37,7 @@ class ReceiptsHandler(BaseHandler): "m.receipt", self._received_remote_receipt ) - # self._earliest_cached_serial = 0 - # self._rooms_to_latest_serial = {} + self._receipt_cache = None @defer.inlineCallbacks def received_client_receipt(self, room_id, receipt_type, user_id, @@ -162,17 +161,9 @@ class ReceiptEventSource(object): rooms = yield self.store.get_rooms_for_user(user.to_string()) rooms = [room.room_id for room in rooms] - events = [] - for room_id in rooms: - content = yield self.store.get_linearized_receipts_for_room( - room_id, from_key, to_key - ) - if content: - events.append({ - "type": "m.receipt", - "room_id": room_id, - "content": content, - }) + events = yield self.store.get_linearized_receipts_for_rooms( + rooms, from_key, to_key + ) defer.returnValue((events, to_key)) @@ -190,16 +181,8 @@ class ReceiptEventSource(object): rooms = yield self.store.get_rooms_for_user(user.to_string()) rooms = [room.room_id for room in rooms] - events = [] - for room_id in rooms: - content = yield self.store.get_linearized_receipts_for_room( - room_id, from_key, to_key - ) - if content: - events.append({ - "type": "m.receipt", - "room_id": room_id, - "content": content, - }) + events = yield self.store.get_linearized_receipts_for_rooms( + rooms, from_key, to_key + ) defer.returnValue((events, to_key)) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 07f8edaac..503f68f85 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -17,6 +17,9 @@ from ._base import SQLBaseStore, cached from twisted.internet import defer +from synapse.util import unwrapFirstError + +from blist import sorteddict import logging @@ -24,6 +27,29 @@ logger = logging.getLogger(__name__) class ReceiptsStore(SQLBaseStore): + def __init__(self, hs): + super(ReceiptsStore, self).__init__(hs) + + self._receipts_stream_cache = _RoomStreamChangeCache() + + @defer.inlineCallbacks + def get_linearized_receipts_for_rooms(self, room_ids, from_key, to_key): + room_ids = set(room_ids) + + if from_key: + room_ids = yield self._receipts_stream_cache.get_rooms_changed( + self, room_ids, from_key + ) + + results = yield defer.gatherResults( + [ + self.get_linearized_receipts_for_room(room_id, from_key, to_key) + for room_id in room_ids + ], + consumeErrors=True, + ).addErrback(unwrapFirstError) + + defer.returnValue([ev for res in results for ev in res]) @defer.inlineCallbacks def get_linearized_receipts_for_room(self, room_id, from_key, to_key): @@ -57,15 +83,22 @@ class ReceiptsStore(SQLBaseStore): "get_linearized_receipts_for_room", f ) - result = {} + if not rows: + defer.returnValue([]) + + content = {} for row in rows: - result.setdefault( + content.setdefault( row["event_id"], {} ).setdefault( row["receipt_type"], [] ).append(row["user_id"]) - defer.returnValue(result) + defer.returnValue([{ + "type": "m.receipt", + "room_id": room_id, + "content": content, + }]) def get_max_receipt_stream_id(self): return self._receipts_id_gen.get_max_token(self) @@ -174,6 +207,9 @@ class ReceiptsStore(SQLBaseStore): stream_id_manager = yield self._receipts_id_gen.get_next(self) with stream_id_manager as stream_id: + yield self._receipts_stream_cache.room_has_changed( + self, room_id, stream_id + ) have_persisted = yield self.runInteraction( "insert_linearized_receipt", self.insert_linearized_receipt_txn, @@ -223,3 +259,53 @@ class ReceiptsStore(SQLBaseStore): for event_id in event_ids ], ) + + +class _RoomStreamChangeCache(object): + """Keeps track of the stream_id of the latest change in rooms. + + Given a list of rooms and stream key, it will give a subset of rooms that + may have changed since that key. If the key is too old then the cache + will simply return all rooms. + """ + def __init__(self, size_of_cache=1000): + self._size_of_cache = size_of_cache + self._room_to_key = {} + self._cache = sorteddict() + self._earliest_key = None + + @defer.inlineCallbacks + def get_rooms_changed(self, store, room_ids, key): + if key > (yield self._get_earliest_key(store)): + keys = self._cache.keys() + i = keys.bisect_right(key) + + result = set( + self._cache[k] for k in keys[i:] + ).intersection(room_ids) + else: + result = room_ids + + defer.returnValue(result) + + @defer.inlineCallbacks + def room_has_changed(self, store, room_id, key): + if key > (yield self._get_earliest_key(store)): + old_key = self._room_to_key.get(room_id, None) + if old_key: + key = max(key, old_key) + self._cache.pop(old_key, None) + self._cache[key] = room_id + + while len(self._cache) > self._size_of_cache: + k, r = self._cache.popitem() + self._earliest_key = max(k, self._earliest_key) + self._room_to_key.pop(r, None) + + @defer.inlineCallbacks + def _get_earliest_key(self, store): + if self._earliest_key is None: + self._earliest_key = yield store.get_max_receipt_stream_id() + self._earliest_key = int(self._earliest_key) + + defer.returnValue(self._earliest_key) From ce9e2f84ad6c62e64884fef1965b786eea3c365c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 8 Jul 2015 15:41:59 +0100 Subject: [PATCH 106/180] Add blist to dependencies --- synapse/python_dependencies.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index f9e59dd91..daaa61b5f 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -31,6 +31,7 @@ REQUIREMENTS = { "pillow": ["PIL"], "pydenticon": ["pydenticon"], "ujson": ["ujson"], + "blist": ["blist"], } CONDITIONAL_REQUIREMENTS = { "web_client": { From 8fb79eeea4b1c388771785024b79e84b4206fc24 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 8 Jul 2015 17:04:29 +0100 Subject: [PATCH 107/180] Only remove one time keys when new one time keys are added --- synapse/storage/end_to_end_keys.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index 936a64669..b3cede37e 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -58,6 +58,11 @@ class EndToEndKeyStore(SQLBaseStore): def add_e2e_one_time_keys(self, user_id, device_id, time_now, valid_until, key_list): def _add_e2e_one_time_keys(txn): + sql = ( + "DELETE FROM e2e_one_time_keys_json" + " WHERE user_id = ? AND device_id = ? AND valid_until_ms < ?" + ) + txn.execute(sql, (user_id, device_id, time_now)) for (algorithm, key_id, json_bytes) in key_list: self._simple_upsert_txn( txn, table="e2e_one_time_keys_json", @@ -83,17 +88,12 @@ class EndToEndKeyStore(SQLBaseStore): Dict mapping from algorithm to number of keys for that algorithm. """ def _count_e2e_one_time_keys(txn): - sql = ( - "DELETE FROM e2e_one_time_keys_json" - " WHERE user_id = ? AND device_id = ? AND valid_until_ms < ?" - ) - txn.execute(sql, (user_id, device_id, time_now)) sql = ( "SELECT algorithm, COUNT(key_id) FROM e2e_one_time_keys_json" - " WHERE user_id = ? AND device_id = ?" + " WHERE user_id = ? AND device_id = ? AND valid_until_ms >= ?" " GROUP BY algorithm" ) - txn.execute(sql, (user_id, device_id)) + txn.execute(sql, (user_id, device_id, time_now)) result = {} for algorithm, key_count in txn.fetchall(): result[algorithm] = key_count From 04192ee05b79310e556753a0029a44aa953c8754 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 8 Jul 2015 17:49:15 +0100 Subject: [PATCH 108/180] typo --- synapse/config/tls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index ecb2d42c1..ad8c5c8ca 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -91,7 +91,7 @@ class TlsConfig(Config): ) if not os.path.exists(tls_certificate_path): - with open(tls_certificate_path, "w") as certifcate_file: + with open(tls_certificate_path, "w") as certificate_file: cert = crypto.X509() subject = cert.get_subject() subject.CN = config["server_name"] @@ -106,7 +106,7 @@ class TlsConfig(Config): cert_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, cert) - certifcate_file.write(cert_pem) + certificate_file.write(cert_pem) if not os.path.exists(tls_dh_params_path): if GENERATE_DH_PARAMS: From 64afbe6ccd19bb2ec94f3fbb3d91586202c924fd Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 8 Jul 2015 18:20:02 +0100 Subject: [PATCH 109/180] add new optional config for tls_certificate_chain_path for folks with intermediary SSL certs --- synapse/config/tls.py | 20 +++++++++++++++++--- synapse/crypto/context_factory.py | 2 ++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index ecb2d42c1..e04fe0d96 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -25,9 +25,19 @@ GENERATE_DH_PARAMS = False class TlsConfig(Config): def read_config(self, config): self.tls_certificate = self.read_tls_certificate( - config.get("tls_certificate_path") + config.get("tls_certificate_path"), + "tls_certificate" ) + tls_certificate_chain_path = + config.get("tls_certificate_chain_path") + + if tls_certificate_chain_path and os.path.exists(tls_certificate_chain_path): + self.tls_certificate_chain = self.read_tls_certificate( + config.get("tls_certificate_chain_path"), + "tls_certificate_chain" + ) + self.no_tls = config.get("no_tls", False) if self.no_tls: @@ -45,6 +55,7 @@ class TlsConfig(Config): base_key_name = os.path.join(config_dir_path, server_name) tls_certificate_path = base_key_name + ".tls.crt" + tls_certificate_chain_path = base_key_name + ".tls.chain.crt" tls_private_key_path = base_key_name + ".tls.key" tls_dh_params_path = base_key_name + ".tls.dh" @@ -52,6 +63,9 @@ class TlsConfig(Config): # PEM encoded X509 certificate for TLS tls_certificate_path: "%(tls_certificate_path)s" + # PEM encoded X509 intermediary certificate file for TLS (optional) + # tls_certificate_chain_path: "%(tls_certificate_chain_path)s" + # PEM encoded private key for TLS tls_private_key_path: "%(tls_private_key_path)s" @@ -62,8 +76,8 @@ class TlsConfig(Config): no_tls: False """ % locals() - def read_tls_certificate(self, cert_path): - cert_pem = self.read_file(cert_path, "tls_certificate") + def read_tls_certificate(self, cert_path, config_name): + cert_pem = self.read_file(cert_path, config_name) return crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) def read_tls_private_key(self, private_key_path): diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 2f8618a0d..ea5dd1e7d 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -38,6 +38,8 @@ class ServerContextFactory(ssl.ContextFactory): logger.exception("Failed to enable eliptic curve for TLS") context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) context.use_certificate(config.tls_certificate) + if config.tls_certificate_chain: + context.use_certificate_chain_file(config.tls_certificate_chain) if not config.no_tls: context.use_privatekey(config.tls_private_key) From 465acb0c6ac575c120c756486171aa701abaa4eb Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 8 Jul 2015 18:30:59 +0100 Subject: [PATCH 110/180] *cough* --- synapse/config/tls.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index e04fe0d96..945af3805 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -29,14 +29,15 @@ class TlsConfig(Config): "tls_certificate" ) - tls_certificate_chain_path = - config.get("tls_certificate_chain_path") + tls_certificate_chain_path = config.get("tls_certificate_chain_path") if tls_certificate_chain_path and os.path.exists(tls_certificate_chain_path): self.tls_certificate_chain = self.read_tls_certificate( config.get("tls_certificate_chain_path"), "tls_certificate_chain" ) + else: + self.tls_certificate_chain = None self.no_tls = config.get("no_tls", False) From 19fa3731aee498159cbc475f8b29f66dacb6aba6 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 8 Jul 2015 18:53:41 +0100 Subject: [PATCH 111/180] typo --- synapse/crypto/context_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index ea5dd1e7d..324dc31fe 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -35,7 +35,7 @@ class ServerContextFactory(ssl.ContextFactory): _ecCurve = _OpenSSLECCurve(_defaultCurveName) _ecCurve.addECKeyToContext(context) except: - logger.exception("Failed to enable eliptic curve for TLS") + logger.exception("Failed to enable elliptic curve for TLS") context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) context.use_certificate(config.tls_certificate) if config.tls_certificate_chain: From f26a3df1bf6cabee28f5f91778082c0f26b2378c Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 8 Jul 2015 21:33:02 +0100 Subject: [PATCH 112/180] oops, context.tls_certificate_chain_file() expects a file, not a certificate. --- synapse/config/tls.py | 5 +---- synapse/crypto/context_factory.py | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 945af3805..5fef63846 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -32,10 +32,7 @@ class TlsConfig(Config): tls_certificate_chain_path = config.get("tls_certificate_chain_path") if tls_certificate_chain_path and os.path.exists(tls_certificate_chain_path): - self.tls_certificate_chain = self.read_tls_certificate( - config.get("tls_certificate_chain_path"), - "tls_certificate_chain" - ) + self.tls_certificate_chain_file = tls_certificate_chain_path else: self.tls_certificate_chain = None diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 324dc31fe..d515007ca 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -38,8 +38,8 @@ class ServerContextFactory(ssl.ContextFactory): logger.exception("Failed to enable elliptic curve for TLS") context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) context.use_certificate(config.tls_certificate) - if config.tls_certificate_chain: - context.use_certificate_chain_file(config.tls_certificate_chain) + if config.tls_certificate_chain_file: + context.use_certificate_chain_file(config.tls_certificate_chain_file) if not config.no_tls: context.use_privatekey(config.tls_private_key) From 8ad2d2d1cb679484dcffc7bacfd9c1de3f6dad38 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 9 Jul 2015 00:06:01 +0100 Subject: [PATCH 113/180] document tls_certificate_chain_path more clearly --- synapse/config/tls.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 5fef63846..de57d0d0e 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -62,6 +62,11 @@ class TlsConfig(Config): tls_certificate_path: "%(tls_certificate_path)s" # PEM encoded X509 intermediary certificate file for TLS (optional) + # This *must* be a concatenation of the tls_certificate pointed to + # by tls_certificate_path followed by the intermediary certificates + # in hierarchical order. If specified this option overrides the + # tls_certificate_path parameter. + # # tls_certificate_chain_path: "%(tls_certificate_chain_path)s" # PEM encoded private key for TLS From fb8d2862c1d7582096b5f8bd6194dcbe8e1afc01 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 9 Jul 2015 00:45:41 +0100 Subject: [PATCH 114/180] remove the tls_certificate_chain_path param and simply support tls_certificate_path pointing to a file containing a chain of certificates --- synapse/config/tls.py | 30 +++++++++--------------------- synapse/crypto/context_factory.py | 4 +--- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index de57d0d0e..e136d1371 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -25,16 +25,9 @@ GENERATE_DH_PARAMS = False class TlsConfig(Config): def read_config(self, config): self.tls_certificate = self.read_tls_certificate( - config.get("tls_certificate_path"), - "tls_certificate" + config.get("tls_certificate_path") ) - - tls_certificate_chain_path = config.get("tls_certificate_chain_path") - - if tls_certificate_chain_path and os.path.exists(tls_certificate_chain_path): - self.tls_certificate_chain_file = tls_certificate_chain_path - else: - self.tls_certificate_chain = None + self.tls_certificate_file = config.get("tls_certificate_path"); self.no_tls = config.get("no_tls", False) @@ -53,22 +46,17 @@ class TlsConfig(Config): base_key_name = os.path.join(config_dir_path, server_name) tls_certificate_path = base_key_name + ".tls.crt" - tls_certificate_chain_path = base_key_name + ".tls.chain.crt" tls_private_key_path = base_key_name + ".tls.key" tls_dh_params_path = base_key_name + ".tls.dh" return """\ - # PEM encoded X509 certificate for TLS + # PEM encoded X509 certificate for TLS. + # You can replace the self-signed certificate that synapse + # autogenerates on launch with your own SSL certificate + key pair + # if you like. Any required intermediary certificates can be + # appended after the primary certificate in hierarchical order. tls_certificate_path: "%(tls_certificate_path)s" - # PEM encoded X509 intermediary certificate file for TLS (optional) - # This *must* be a concatenation of the tls_certificate pointed to - # by tls_certificate_path followed by the intermediary certificates - # in hierarchical order. If specified this option overrides the - # tls_certificate_path parameter. - # - # tls_certificate_chain_path: "%(tls_certificate_chain_path)s" - # PEM encoded private key for TLS tls_private_key_path: "%(tls_private_key_path)s" @@ -79,8 +67,8 @@ class TlsConfig(Config): no_tls: False """ % locals() - def read_tls_certificate(self, cert_path, config_name): - cert_pem = self.read_file(cert_path, config_name) + def read_tls_certificate(self, cert_path): + cert_pem = self.read_file(cert_path, "tls_certificate") return crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) def read_tls_private_key(self, private_key_path): diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index d515007ca..c4390f3b2 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -37,9 +37,7 @@ class ServerContextFactory(ssl.ContextFactory): except: logger.exception("Failed to enable elliptic curve for TLS") context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) - context.use_certificate(config.tls_certificate) - if config.tls_certificate_chain_file: - context.use_certificate_chain_file(config.tls_certificate_chain_file) + context.use_certificate_chain_file(config.tls_certificate_file) if not config.no_tls: context.use_privatekey(config.tls_private_key) From d2caa5351aece72b274f78fe81348f715389d421 Mon Sep 17 00:00:00 2001 From: Muthu Subramanian Date: Thu, 9 Jul 2015 12:58:15 +0530 Subject: [PATCH 115/180] code beautify --- synapse/rest/client/v1/login.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index b4c74c4c2..b4894497b 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -20,19 +20,15 @@ from synapse.types import UserID from base import ClientV1RestServlet, client_path_pattern import simplejson as json -import cgi import urllib import logging -from saml2 import BINDING_HTTP_REDIRECT from saml2 import BINDING_HTTP_POST -from saml2.metadata import create_metadata_string from saml2 import config from saml2.client import Saml2Client -from saml2.httputil import ServiceError -from saml2.samlp import Extensions -from saml2.extension.pefim import SPCertEnc -from saml2.s_utils import rndstr + + +logger = logging.getLogger(__name__) class LoginRestServlet(ClientV1RestServlet): @@ -137,9 +133,8 @@ class SAML2RestServlet(ClientV1RestServlet): conf.load_file(self.sp_config) SP = Saml2Client(conf) saml2_auth = SP.parse_authn_request_response( - request.args['SAMLResponse'][0], BINDING_HTTP_POST) + request.args['SAMLResponse'][0], BINDING_HTTP_POST) except Exception, e: # Not authenticated - logger = logging.getLogger(__name__) logger.exception(e) if saml2_auth and saml2_auth.status_ok() and not saml2_auth.not_signed: username = saml2_auth.name_id.text From 8cd34dfe955841d7ff3306b84a686e7138aec526 Mon Sep 17 00:00:00 2001 From: Muthu Subramanian Date: Thu, 9 Jul 2015 13:34:47 +0530 Subject: [PATCH 116/180] Make SAML2 optional and add some references/comments --- synapse/config/saml2.py | 14 ++++++++++++++ synapse/rest/client/v1/login.py | 13 +++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/synapse/config/saml2.py b/synapse/config/saml2.py index d18d076a8..be5176db5 100644 --- a/synapse/config/saml2.py +++ b/synapse/config/saml2.py @@ -16,6 +16,19 @@ from ._base import Config +# +# SAML2 Configuration +# Synapse uses pysaml2 libraries for providing SAML2 support +# +# config_path: Path to the sp_conf.py configuration file +# idp_redirect_url: Identity provider URL which will redirect +# the user back to /login/saml2 with proper info. +# +# sp_conf.py file is something like: +# https://github.com/rohe/pysaml2/blob/master/example/sp-repoze/sp_conf.py.example +# +# More information: https://pythonhosted.org/pysaml2/howto/config.html +# class SAML2Config(Config): def read_config(self, config): self.saml2_config = config["saml2_config"] @@ -23,6 +36,7 @@ class SAML2Config(Config): def default_config(self, config_dir_path, server_name): return """ saml2_config: + enabled: false config_path: "%s/sp_conf.py" idp_redirect_url: "http://%s/idp" """ % (config_dir_path, server_name) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index b4894497b..f64f5e990 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -39,10 +39,13 @@ class LoginRestServlet(ClientV1RestServlet): def __init__(self, hs): super(LoginRestServlet, self).__init__(hs) self.idp_redirect_url = hs.config.saml2_config['idp_redirect_url'] + self.saml2_enabled = hs.config.saml2_config['enabled'] def on_GET(self, request): - return (200, {"flows": [{"type": LoginRestServlet.PASS_TYPE}, - {"type": LoginRestServlet.SAML2_TYPE}]}) + flows = [{"type": LoginRestServlet.PASS_TYPE}] + if self.saml2_enabled: + flows.append({"type": LoginRestServlet.SAML2_TYPE}) + return (200, {"flows": flows}) def on_OPTIONS(self, request): return (200, {}) @@ -54,7 +57,8 @@ class LoginRestServlet(ClientV1RestServlet): if login_submission["type"] == LoginRestServlet.PASS_TYPE: result = yield self.do_password_login(login_submission) defer.returnValue(result) - elif login_submission["type"] == LoginRestServlet.SAML2_TYPE: + elif self.saml2_enabled and (login_submission["type"] == + LoginRestServlet.SAML2_TYPE): relay_state = "" if "relay_state" in login_submission: relay_state = "&RelayState="+urllib.quote( @@ -173,5 +177,6 @@ def _parse_json(request): def register_servlets(hs, http_server): LoginRestServlet(hs).register(http_server) - SAML2RestServlet(hs).register(http_server) + if hs.config.saml2_config['enabled']: + SAML2RestServlet(hs).register(http_server) # TODO PasswordResetRestServlet(hs).register(http_server) From 1af188209a03567dc4b5300b9a0fc8613ad176df Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Jul 2015 11:39:30 +0100 Subject: [PATCH 117/180] Change format of receipts to allow inclusion of data --- synapse/handlers/receipts.py | 24 ++++++++---- synapse/storage/receipts.py | 39 ++++++++++---------- synapse/storage/schema/delta/21/receipts.sql | 16 +++----- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 053ed8480..8a052f071 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -36,6 +36,7 @@ class ReceiptsHandler(BaseHandler): self.federation.register_edu_handler( "m.receipt", self._received_remote_receipt ) + self.clock = self.hs.get_clock() self._receipt_cache = None @@ -51,6 +52,9 @@ class ReceiptsHandler(BaseHandler): "receipt_type": receipt_type, "user_id": user_id, "event_ids": [event_id], + "data": { + "ts": self.clock.time_msec() + } } is_new = yield self._handle_new_receipts([receipt]) @@ -65,12 +69,12 @@ class ReceiptsHandler(BaseHandler): "room_id": room_id, "receipt_type": receipt_type, "user_id": user_id, - "event_ids": [event_id], + "event_ids": user_values["event_ids"], + "data": user_values.get("data", {}), } for room_id, room_values in content.items() - for event_id, ev_values in room_values.items() - for receipt_type, users in ev_values.items() - for user_id in users + for receipt_type, users in room_values.items() + for user_id, user_values in users.items() ] yield self._handle_new_receipts(receipts) @@ -82,9 +86,10 @@ class ReceiptsHandler(BaseHandler): receipt_type = receipt["receipt_type"] user_id = receipt["user_id"] event_ids = receipt["event_ids"] + data = receipt["data"] res = yield self.store.insert_receipt( - room_id, receipt_type, user_id, event_ids, + room_id, receipt_type, user_id, event_ids, data ) if not res: @@ -108,6 +113,7 @@ class ReceiptsHandler(BaseHandler): receipt_type = receipt["receipt_type"] user_id = receipt["user_id"] event_ids = receipt["event_ids"] + data = receipt["data"] remotedomains = set() @@ -124,10 +130,12 @@ class ReceiptsHandler(BaseHandler): edu_type="m.receipt", content={ room_id: { - event_id: { - receipt_type: [user_id] + receipt_type: { + user_id: { + "event_ids": event_ids, + "data": data, + } } - for event_id in event_ids }, }, ) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 503f68f85..c4e6b02bd 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -21,6 +21,7 @@ from synapse.util import unwrapFirstError from blist import sorteddict import logging +import ujson as json logger = logging.getLogger(__name__) @@ -91,8 +92,8 @@ class ReceiptsStore(SQLBaseStore): content.setdefault( row["event_id"], {} ).setdefault( - row["receipt_type"], [] - ).append(row["user_id"]) + row["receipt_type"], {} + )[row["user_id"]] = json.loads(row["data"]) defer.returnValue([{ "type": "m.receipt", @@ -124,7 +125,7 @@ class ReceiptsStore(SQLBaseStore): defer.returnValue(result) def insert_linearized_receipt_txn(self, txn, room_id, receipt_type, - user_id, event_id, stream_id): + user_id, event_id, data, stream_id): # We don't want to clobber receipts for more recent events, so we # have to compare orderings of existing receipts @@ -172,13 +173,14 @@ class ReceiptsStore(SQLBaseStore): "receipt_type": receipt_type, "user_id": user_id, "event_id": event_id, + "data": json.dumps(data), } ) return True @defer.inlineCallbacks - def insert_receipt(self, room_id, receipt_type, user_id, event_ids): + def insert_receipt(self, room_id, receipt_type, user_id, event_ids, data): if not event_ids: return @@ -214,6 +216,7 @@ class ReceiptsStore(SQLBaseStore): "insert_linearized_receipt", self.insert_linearized_receipt_txn, room_id, receipt_type, user_id, linearized_event_id, + data, stream_id=stream_id, ) @@ -221,22 +224,22 @@ class ReceiptsStore(SQLBaseStore): defer.returnValue(None) yield self.insert_graph_receipt( - room_id, receipt_type, user_id, event_ids + room_id, receipt_type, user_id, event_ids, data ) max_persisted_id = yield self._stream_id_gen.get_max_token(self) defer.returnValue((stream_id, max_persisted_id)) - def insert_graph_receipt(self, room_id, receipt_type, - user_id, event_ids): + def insert_graph_receipt(self, room_id, receipt_type, user_id, event_ids, + data): return self.runInteraction( "insert_graph_receipt", self.insert_graph_receipt_txn, - room_id, receipt_type, user_id, event_ids, + room_id, receipt_type, user_id, event_ids, data ) def insert_graph_receipt_txn(self, txn, room_id, receipt_type, - user_id, event_ids): + user_id, event_ids, data): self._simple_delete_txn( txn, table="receipts_graph", @@ -246,18 +249,16 @@ class ReceiptsStore(SQLBaseStore): "user_id": user_id, } ) - self._simple_insert_many_txn( + self._simple_insert_txn( txn, table="receipts_graph", - values=[ - { - "room_id": room_id, - "receipt_type": receipt_type, - "user_id": user_id, - "event_id": event_id, - } - for event_id in event_ids - ], + values={ + "room_id": room_id, + "receipt_type": receipt_type, + "user_id": user_id, + "event_ids": json.dumps(event_ids), + "data": json.dumps(data), + } ) diff --git a/synapse/storage/schema/delta/21/receipts.sql b/synapse/storage/schema/delta/21/receipts.sql index ac7738e37..2f64d609f 100644 --- a/synapse/storage/schema/delta/21/receipts.sql +++ b/synapse/storage/schema/delta/21/receipts.sql @@ -18,11 +18,9 @@ CREATE TABLE IF NOT EXISTS receipts_graph( room_id TEXT NOT NULL, receipt_type TEXT NOT NULL, user_id TEXT NOT NULL, - event_id TEXT NOT NULL -); - -CREATE INDEX receipts_graph_room_tuple ON receipts_graph( - room_id, receipt_type, user_id + event_ids TEXT NOT NULL, + data TEXT NOT NULL, + CONSTRAINT receipts_graph_uniqueness UNIQUE (room_id, receipt_type, user_id) ); CREATE TABLE IF NOT EXISTS receipts_linearized ( @@ -30,11 +28,9 @@ CREATE TABLE IF NOT EXISTS receipts_linearized ( room_id TEXT NOT NULL, receipt_type TEXT NOT NULL, user_id TEXT NOT NULL, - event_id TEXT NOT NULL -); - -CREATE INDEX receipts_linearized_room_tuple ON receipts_linearized( - room_id, receipt_type, user_id + event_id TEXT NOT NULL, + data TEXT NOT NULL, + CONSTRAINT receipts_linearized_uniqueness UNIQUE (room_id, receipt_type, user_id) ); CREATE INDEX receipts_linearized_id ON receipts_linearized( From 294dbd712fee435489fe6e9f9942bd9c39fd480c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Jul 2015 11:47:24 +0100 Subject: [PATCH 118/180] We don't want semicolons. --- synapse/config/tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index e136d1371..6c1df35e8 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -27,7 +27,7 @@ class TlsConfig(Config): self.tls_certificate = self.read_tls_certificate( config.get("tls_certificate_path") ) - self.tls_certificate_file = config.get("tls_certificate_path"); + self.tls_certificate_file = config.get("tls_certificate_path") self.no_tls = config.get("no_tls", False) From c2d08ca62aebdfa689ae7d39ff74b2a43e4b5e3a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Jul 2015 13:15:34 +0100 Subject: [PATCH 119/180] Integer timestamps --- synapse/handlers/receipts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 8a052f071..403c1c849 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -53,7 +53,7 @@ class ReceiptsHandler(BaseHandler): "user_id": user_id, "event_ids": [event_id], "data": { - "ts": self.clock.time_msec() + "ts": int(self.clock.time_msec()), } } From bf0d59ed30b63c6a355e7b3f2a74a26181fd6893 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 9 Jul 2015 14:04:03 +0100 Subject: [PATCH 120/180] Don't bother with a timeout for one time keys on the server. --- synapse/rest/client/v2_alpha/keys.py | 25 ++++++------------- synapse/storage/end_to_end_keys.py | 20 +++++---------- .../schema/delta/21/end_to_end_keys.sql | 1 - 3 files changed, 13 insertions(+), 33 deletions(-) diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/v2_alpha/keys.py index 3bb4ad64f..4b617c251 100644 --- a/synapse/rest/client/v2_alpha/keys.py +++ b/synapse/rest/client/v2_alpha/keys.py @@ -50,7 +50,6 @@ class KeyUploadServlet(RestServlet): "one_time_keys": { ":": "" }, - "one_time_keys_valid_for": , } """ PATTERN = client_v2_pattern("/keys/upload/(?P[^/]*)") @@ -87,13 +86,10 @@ class KeyUploadServlet(RestServlet): ) one_time_keys = body.get("one_time_keys", None) - one_time_keys_valid_for = body.get("one_time_keys_valid_for", None) if one_time_keys: - valid_until = int(one_time_keys_valid_for) + time_now logger.info( - "Adding %d one_time_keys for device %r for user %r at %d" - " valid_until %d", - len(one_time_keys), device_id, user_id, time_now, valid_until + "Adding %d one_time_keys for device %r for user %r at %d", + len(one_time_keys), device_id, user_id, time_now ) key_list = [] for key_id, key_json in one_time_keys.items(): @@ -103,23 +99,18 @@ class KeyUploadServlet(RestServlet): )) yield self.store.add_e2e_one_time_keys( - user_id, device_id, time_now, valid_until, key_list + user_id, device_id, time_now, key_list ) - result = yield self.store.count_e2e_one_time_keys( - user_id, device_id, time_now - ) + result = yield self.store.count_e2e_one_time_keys(user_id, device_id) defer.returnValue((200, {"one_time_key_counts": result})) @defer.inlineCallbacks def on_GET(self, request, device_id): auth_user, client_info = yield self.auth.get_user_by_req(request) user_id = auth_user.to_string() - time_now = self.clock.time_msec() - result = yield self.store.count_e2e_one_time_keys( - user_id, device_id, time_now - ) + result = yield self.store.count_e2e_one_time_keys(user_id, device_id) defer.returnValue((200, {"one_time_key_counts": result})) @@ -249,9 +240,8 @@ class OneTimeKeyServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id, device_id, algorithm): yield self.auth.get_user_by_req(request) - time_now = self.clock.time_msec() results = yield self.store.take_e2e_one_time_keys( - [(user_id, device_id, algorithm)], time_now + [(user_id, device_id, algorithm)] ) defer.returnValue(self.json_result(request, results)) @@ -266,8 +256,7 @@ class OneTimeKeyServlet(RestServlet): for user_id, device_keys in body.get("one_time_keys", {}).items(): for device_id, algorithm in device_keys.items(): query.append((user_id, device_id, algorithm)) - time_now = self.clock.time_msec() - results = yield self.store.take_e2e_one_time_keys(query, time_now) + results = yield self.store.take_e2e_one_time_keys(query) defer.returnValue(self.json_result(request, results)) def json_result(self, request, results): diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index b3cede37e..99dc864e4 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -55,14 +55,8 @@ class EndToEndKeyStore(SQLBaseStore): return result return self.runInteraction("get_e2e_device_keys", _get_e2e_device_keys) - def add_e2e_one_time_keys(self, user_id, device_id, time_now, valid_until, - key_list): + def add_e2e_one_time_keys(self, user_id, device_id, time_now, key_list): def _add_e2e_one_time_keys(txn): - sql = ( - "DELETE FROM e2e_one_time_keys_json" - " WHERE user_id = ? AND device_id = ? AND valid_until_ms < ?" - ) - txn.execute(sql, (user_id, device_id, time_now)) for (algorithm, key_id, json_bytes) in key_list: self._simple_upsert_txn( txn, table="e2e_one_time_keys_json", @@ -74,7 +68,6 @@ class EndToEndKeyStore(SQLBaseStore): }, values={ "ts_added_ms": time_now, - "valid_until_ms": valid_until, "key_json": json_bytes, } ) @@ -82,7 +75,7 @@ class EndToEndKeyStore(SQLBaseStore): "add_e2e_one_time_keys", _add_e2e_one_time_keys ) - def count_e2e_one_time_keys(self, user_id, device_id, time_now): + def count_e2e_one_time_keys(self, user_id, device_id): """ Count the number of one time keys the server has for a device Returns: Dict mapping from algorithm to number of keys for that algorithm. @@ -90,10 +83,10 @@ class EndToEndKeyStore(SQLBaseStore): def _count_e2e_one_time_keys(txn): sql = ( "SELECT algorithm, COUNT(key_id) FROM e2e_one_time_keys_json" - " WHERE user_id = ? AND device_id = ? AND valid_until_ms >= ?" + " WHERE user_id = ? AND device_id = ?" " GROUP BY algorithm" ) - txn.execute(sql, (user_id, device_id, time_now)) + txn.execute(sql, (user_id, device_id)) result = {} for algorithm, key_count in txn.fetchall(): result[algorithm] = key_count @@ -102,13 +95,12 @@ class EndToEndKeyStore(SQLBaseStore): "count_e2e_one_time_keys", _count_e2e_one_time_keys ) - def take_e2e_one_time_keys(self, query_list, time_now): + def take_e2e_one_time_keys(self, query_list): """Take a list of one time keys out of the database""" def _take_e2e_one_time_keys(txn): sql = ( "SELECT key_id, key_json FROM e2e_one_time_keys_json" " WHERE user_id = ? AND device_id = ? AND algorithm = ?" - " AND valid_until_ms > ?" " LIMIT 1" ) result = {} @@ -116,7 +108,7 @@ class EndToEndKeyStore(SQLBaseStore): for user_id, device_id, algorithm in query_list: user_result = result.setdefault(user_id, {}) device_result = user_result.setdefault(device_id, {}) - txn.execute(sql, (user_id, device_id, algorithm, time_now)) + txn.execute(sql, (user_id, device_id, algorithm)) for key_id, key_json in txn.fetchall(): device_result[algorithm + ":" + key_id] = key_json delete.append((user_id, device_id, algorithm, key_id)) diff --git a/synapse/storage/schema/delta/21/end_to_end_keys.sql b/synapse/storage/schema/delta/21/end_to_end_keys.sql index 107d2e67c..95e27eb7e 100644 --- a/synapse/storage/schema/delta/21/end_to_end_keys.sql +++ b/synapse/storage/schema/delta/21/end_to_end_keys.sql @@ -29,7 +29,6 @@ CREATE TABLE IF NOT EXISTS e2e_one_time_keys_json ( algorithm TEXT NOT NULL, -- Which algorithm this one-time key is for. key_id TEXT NOT NULL, -- An id for suppressing duplicate uploads. ts_added_ms BIGINT NOT NULL, -- When this key was uploaded. - valid_until_ms BIGINT NOT NULL, -- When this key is valid until. key_json TEXT NOT NULL, -- The key as a JSON blob. CONSTRAINT uniqueness UNIQUE (user_id, device_id, algorithm, key_id) ); From f0979afdb0b0b8f96730a97cf0b51de6024cabda Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Jul 2015 16:02:07 +0100 Subject: [PATCH 121/180] Remove spurious comment --- synapse/handlers/receipts.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 403c1c849..f847360d0 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Contains handlers for federation events.""" - from ._base import BaseHandler from twisted.internet import defer From ed887209520cc3268885c918738734f8b9874c81 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Jul 2015 16:14:46 +0100 Subject: [PATCH 122/180] Handle error slightly better --- synapse/storage/receipts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index c4e6b02bd..593032713 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -188,6 +188,7 @@ class ReceiptsStore(SQLBaseStore): linearized_event_id = event_ids[0] else: # we need to points in graph -> linearized form. + # TODO: Make this better. def graph_to_linear(txn): query = ( "SELECT event_id WHERE room_id = ? AND stream_ordering IN (" @@ -200,8 +201,7 @@ class ReceiptsStore(SQLBaseStore): if rows: return rows[0][0] else: - # TODO: ARGH?! - return None + raise RuntimeError("Unrecognized event_ids: %r" % (event_ids,)) linearized_event_id = yield self.runInteraction( "insert_receipt_conv", graph_to_linear From b5f0d73ea3f6611c0980a03a0dfe57058071013e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 9 Jul 2015 17:09:26 +0100 Subject: [PATCH 123/180] Add comment --- synapse/handlers/federation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index cd3867ed9..d7f197f24 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -242,6 +242,10 @@ class FederationHandler(BaseHandler): if history: visibility = history.content.get("history_visibility", "shared") if visibility in ["invited", "joined"]: + # We now loop through all state events looking for + # membership states for the requesting server to determine + # if the server is either in the room or has been invited + # into the room. for ev in state.values(): if ev.type != EventTypes.Member: continue From a887efa07aa9757538cb36d0dfa2b79925eba4aa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Jul 2015 10:37:24 +0100 Subject: [PATCH 124/180] Add Muthu Subramanian to AUTHORS --- AUTHORS.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index d7224ff5d..54ced6700 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -42,3 +42,6 @@ Ivan Shapovalov Eric Myhre * Fix bug where ``media_store_path`` config option was ignored by v0 content repository API. + +Muthu Subramanian + * Add SAML2 support for registration and logins. From f3049d0b81ad626de7ca80330608b374e0ec8b5b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Jul 2015 10:50:03 +0100 Subject: [PATCH 125/180] Small tweaks to SAML2 configuration. - Add saml2 config docs to default config. - Use existence of saml2 config to indicate if saml2 should be enabled. --- synapse/config/saml2.py | 48 ++++++++++++++++++++------------- synapse/rest/client/v1/login.py | 8 +++--- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/synapse/config/saml2.py b/synapse/config/saml2.py index be5176db5..153203687 100644 --- a/synapse/config/saml2.py +++ b/synapse/config/saml2.py @@ -16,27 +16,39 @@ from ._base import Config -# -# SAML2 Configuration -# Synapse uses pysaml2 libraries for providing SAML2 support -# -# config_path: Path to the sp_conf.py configuration file -# idp_redirect_url: Identity provider URL which will redirect -# the user back to /login/saml2 with proper info. -# -# sp_conf.py file is something like: -# https://github.com/rohe/pysaml2/blob/master/example/sp-repoze/sp_conf.py.example -# -# More information: https://pythonhosted.org/pysaml2/howto/config.html -# class SAML2Config(Config): + """SAML2 Configuration + Synapse uses pysaml2 libraries for providing SAML2 support + + config_path: Path to the sp_conf.py configuration file + idp_redirect_url: Identity provider URL which will redirect + the user back to /login/saml2 with proper info. + + sp_conf.py file is something like: + https://github.com/rohe/pysaml2/blob/master/example/sp-repoze/sp_conf.py.example + + More information: https://pythonhosted.org/pysaml2/howto/config.html + """ + def read_config(self, config): - self.saml2_config = config["saml2_config"] + saml2_config = config.get("saml2_config", None) + if saml2_config: + self.saml2_enabled = True + self.saml2_config_path = saml2_config["config_path"] + self.saml2_idp_redirect_url = saml2_config["idp_redirect_url"] + else: + self.saml2_enabled = False + self.saml2_config_path = None + self.saml2_idp_redirect_url = None def default_config(self, config_dir_path, server_name): return """ - saml2_config: - enabled: false - config_path: "%s/sp_conf.py" - idp_redirect_url: "http://%s/idp" + # Enable SAML2 for registration and login. Uses pysaml2 + # config_path: Path to the sp_conf.py configuration file + # idp_redirect_url: Identity provider URL which will redirect + # the user back to /login/saml2 with proper info. + # See pysaml2 docs for format of config. + #saml2_config: + # config_path: "%s/sp_conf.py" + # idp_redirect_url: "http://%s/idp" """ % (config_dir_path, server_name) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index f64f5e990..998d4d44c 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -38,8 +38,8 @@ class LoginRestServlet(ClientV1RestServlet): def __init__(self, hs): super(LoginRestServlet, self).__init__(hs) - self.idp_redirect_url = hs.config.saml2_config['idp_redirect_url'] - self.saml2_enabled = hs.config.saml2_config['enabled'] + self.idp_redirect_url = hs.config.saml2_idp_redirect_url + self.saml2_enabled = hs.config.saml2_enabled def on_GET(self, request): flows = [{"type": LoginRestServlet.PASS_TYPE}] @@ -127,7 +127,7 @@ class SAML2RestServlet(ClientV1RestServlet): def __init__(self, hs): super(SAML2RestServlet, self).__init__(hs) - self.sp_config = hs.config.saml2_config['config_path'] + self.sp_config = hs.config.saml2_config_path @defer.inlineCallbacks def on_POST(self, request): @@ -177,6 +177,6 @@ def _parse_json(request): def register_servlets(hs, http_server): LoginRestServlet(hs).register(http_server) - if hs.config.saml2_config['enabled']: + if hs.config.saml2_enabled: SAML2RestServlet(hs).register(http_server) # TODO PasswordResetRestServlet(hs).register(http_server) From a01097d60b9c711d71cd5d1c63cb4fb5b95a8a63 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 10 Jul 2015 13:26:18 +0100 Subject: [PATCH 126/180] Assume that each device for a user has only one of each type of key --- synapse/rest/client/v2_alpha/keys.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/v2_alpha/keys.py index 4b617c251..f03126775 100644 --- a/synapse/rest/client/v2_alpha/keys.py +++ b/synapse/rest/client/v2_alpha/keys.py @@ -41,11 +41,11 @@ class KeyUploadServlet(RestServlet): "m.olm.curve25519-aes-sha256", ] "keys": { - ":": "", + ":": "", }, "signatures:" { - "/" { - ":": "" + "" { + ":": "" } } }, "one_time_keys": { ":": "" From 0d7f0febf45575666d182cb2ad0cf451b11fbf07 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Jul 2015 13:43:03 +0100 Subject: [PATCH 127/180] Uniquely name unique constraint --- synapse/storage/schema/delta/21/end_to_end_keys.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/schema/delta/21/end_to_end_keys.sql b/synapse/storage/schema/delta/21/end_to_end_keys.sql index 95e27eb7e..8b4a380d1 100644 --- a/synapse/storage/schema/delta/21/end_to_end_keys.sql +++ b/synapse/storage/schema/delta/21/end_to_end_keys.sql @@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS e2e_device_keys_json ( device_id TEXT NOT NULL, -- Which of the user's devices these keys are for. ts_added_ms BIGINT NOT NULL, -- When the keys were uploaded. key_json TEXT NOT NULL, -- The keys for the device as a JSON blob. - CONSTRAINT uniqueness UNIQUE (user_id, device_id) + CONSTRAINT e2e_device_keys_json_uniqueness UNIQUE (user_id, device_id) ); @@ -30,5 +30,5 @@ CREATE TABLE IF NOT EXISTS e2e_one_time_keys_json ( key_id TEXT NOT NULL, -- An id for suppressing duplicate uploads. ts_added_ms BIGINT NOT NULL, -- When this key was uploaded. key_json TEXT NOT NULL, -- The key as a JSON blob. - CONSTRAINT uniqueness UNIQUE (user_id, device_id, algorithm, key_id) + CONSTRAINT e2e_one_time_keys_json_uniqueness UNIQUE (user_id, device_id, algorithm, key_id) ); From 7e3b14fe782eedbe37a0eb8c17da605d2373e594 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Jul 2015 13:21:31 +0100 Subject: [PATCH 128/180] You shouldn't be able to ban/kick users with higher power levels --- synapse/api/auth.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 4da62e5d8..bd2f058e4 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -187,6 +187,9 @@ class Auth(object): join_rule = JoinRules.INVITE user_level = self._get_user_power_level(event.user_id, auth_events) + target_level = self._get_user_power_level( + target_user_id, auth_events + ) # FIXME (erikj): What should we do here as the default? ban_level = self._get_named_level(auth_events, "ban", 50) @@ -258,12 +261,12 @@ class Auth(object): elif target_user_id != event.user_id: kick_level = self._get_named_level(auth_events, "kick", 50) - if user_level < kick_level: + if user_level < kick_level or user_level < target_level: raise AuthError( 403, "You cannot kick user %s." % target_user_id ) elif Membership.BAN == membership: - if user_level < ban_level: + if user_level < ban_level or user_level < target_level: raise AuthError(403, "You don't have permission to ban") else: raise AuthError(500, "Unknown membership %s" % membership) From a5ea22d4683ea890cb4c4ba000502f116814dd1d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Jul 2015 13:42:24 +0100 Subject: [PATCH 129/180] Sanitize power level checks --- synapse/api/auth.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index bd2f058e4..ee52ff66d 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -261,12 +261,12 @@ class Auth(object): elif target_user_id != event.user_id: kick_level = self._get_named_level(auth_events, "kick", 50) - if user_level < kick_level or user_level < target_level: + if user_level < kick_level or user_level <= target_level: raise AuthError( 403, "You cannot kick user %s." % target_user_id ) elif Membership.BAN == membership: - if user_level < ban_level or user_level < target_level: + if user_level < ban_level or user_level <= target_level: raise AuthError(403, "You don't have permission to ban") else: raise AuthError(500, "Unknown membership %s" % membership) @@ -576,25 +576,25 @@ class Auth(object): # Check other levels: levels_to_check = [ - ("users_default", []), - ("events_default", []), - ("ban", []), - ("redact", []), - ("kick", []), - ("invite", []), + ("users_default", None), + ("events_default", None), + ("ban", None), + ("redact", None), + ("kick", None), + ("invite", None), ] old_list = current_state.content.get("users") for user in set(old_list.keys() + user_list.keys()): levels_to_check.append( - (user, ["users"]) + (user, "users") ) old_list = current_state.content.get("events") new_list = event.content.get("events") for ev_id in set(old_list.keys() + new_list.keys()): levels_to_check.append( - (ev_id, ["events"]) + (ev_id, "events") ) old_state = current_state.content @@ -602,12 +602,10 @@ class Auth(object): for level_to_check, dir in levels_to_check: old_loc = old_state - for d in dir: - old_loc = old_loc.get(d, {}) - new_loc = new_state - for d in dir: - new_loc = new_loc.get(d, {}) + if dir: + old_loc = old_loc.get(dir, {}) + new_loc = new_loc.get(dir, {}) if level_to_check in old_loc: old_level = int(old_loc[level_to_check]) @@ -623,6 +621,14 @@ class Auth(object): if new_level == old_level: continue + if dir == "users" and level_to_check != event.user_id: + if old_level == user_level: + raise AuthError( + 403, + "You don't have permission to remove ops level equal " + "to your own" + ) + if old_level > user_level or new_level > user_level: raise AuthError( 403, From 17bb9a7eb93cdc5534b63d5c2ccf2d978b2bec84 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 10 Jul 2015 14:07:57 +0100 Subject: [PATCH 130/180] Remove commented out code --- synapse/storage/events.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index d760acb87..ed7ea3880 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -366,20 +366,6 @@ class EventsStore(SQLBaseStore): txn, event.event_id, context.rejected ) - # for hash_alg, hash_base64 in event.hashes.items(): - # hash_bytes = decode_base64(hash_base64) - # self._store_event_content_hash_txn( - # txn, event.event_id, hash_alg, hash_bytes, - # ) - - # for prev_event_id, prev_hashes in event.prev_events: - # for alg, hash_base64 in prev_hashes.items(): - # hash_bytes = decode_base64(hash_base64) - # self._store_prev_event_hash_txn( - # txn, event.event_id, prev_event_id, alg, - # hash_bytes - # ) - self._simple_insert_many_txn( txn, table="event_auth", From e5991af629df2e63c20c5f10e4589a9faf8305cb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Jul 2015 13:30:43 +0100 Subject: [PATCH 131/180] Comments --- synapse/handlers/receipts.py | 16 ++++++++++++---- synapse/storage/receipts.py | 11 +++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index f847360d0..1925a4803 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -41,10 +41,9 @@ class ReceiptsHandler(BaseHandler): @defer.inlineCallbacks def received_client_receipt(self, room_id, receipt_type, user_id, event_id): - # 1. Persist. - # 2. Notify local clients - # 3. Notify remote servers - + """Called when a client tells us a local user has read up to the given + event_id in the room. + """ receipt = { "room_id": room_id, "receipt_type": receipt_type, @@ -62,6 +61,8 @@ class ReceiptsHandler(BaseHandler): @defer.inlineCallbacks def _received_remote_receipt(self, origin, content): + """Called when we receive an EDU of type m.receipt from a remote HS. + """ receipts = [ { "room_id": room_id, @@ -79,6 +80,8 @@ class ReceiptsHandler(BaseHandler): @defer.inlineCallbacks def _handle_new_receipts(self, receipts): + """Takes a list of receipts, stores them and informs the notifier. + """ for receipt in receipts: room_id = receipt["room_id"] receipt_type = receipt["receipt_type"] @@ -105,6 +108,9 @@ class ReceiptsHandler(BaseHandler): @defer.inlineCallbacks def _push_remotes(self, receipts): + """Given a list of receipts, works out which remote servers should be + poked and pokes them. + """ # TODO: Some of this stuff should be coallesced. for receipt in receipts: room_id = receipt["room_id"] @@ -140,6 +146,8 @@ class ReceiptsHandler(BaseHandler): @defer.inlineCallbacks def get_receipts_for_room(self, room_id, to_key): + """Gets all receipts for a room, upto the given key. + """ result = yield self.store.get_linearized_receipts_for_room( room_id, None, to_key ) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 593032713..56b9fedfd 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -35,6 +35,8 @@ class ReceiptsStore(SQLBaseStore): @defer.inlineCallbacks def get_linearized_receipts_for_rooms(self, room_ids, from_key, to_key): + """Get receipts for multiple rooms for sending to clients. + """ room_ids = set(room_ids) if from_key: @@ -54,6 +56,8 @@ class ReceiptsStore(SQLBaseStore): @defer.inlineCallbacks def get_linearized_receipts_for_room(self, room_id, from_key, to_key): + """Get receipts for a single room for sending to clients. + """ def f(txn): if from_key: sql = ( @@ -107,6 +111,8 @@ class ReceiptsStore(SQLBaseStore): @cached @defer.inlineCallbacks def get_graph_receipts_for_room(self, room_id): + """Get receipts for sending to remote servers. + """ rows = yield self._simple_select_list( table="receipts_graph", keyvalues={"room_id": room_id}, @@ -181,6 +187,11 @@ class ReceiptsStore(SQLBaseStore): @defer.inlineCallbacks def insert_receipt(self, room_id, receipt_type, user_id, event_ids, data): + """Insert a receipt, either from local client or remote server. + + Automatically does conversion between linearized and graph + representations. + """ if not event_ids: return From d5cc7945985e4cbf17c7dbf4c7c45071b87d030e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 13 Jul 2015 16:48:06 +0100 Subject: [PATCH 132/180] Implement presets at room creation --- synapse/api/constants.py | 5 +++ synapse/handlers/room.py | 82 +++++++++++++++++++++++++++++----------- 2 files changed, 65 insertions(+), 22 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 3e15e8a9d..885b9c359 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -87,3 +87,8 @@ class RejectedReason(object): AUTH_ERROR = "auth_error" REPLACED = "replaced" NOT_ANCESTOR = "not_ancestor" + + +class RoomCreationPreset(object): + PrivateChat = "private_chat" + PublicChat = "public_chat" diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 891707df4..c65439543 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -19,7 +19,9 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.types import UserID, RoomAlias, RoomID -from synapse.api.constants import EventTypes, Membership, JoinRules +from synapse.api.constants import ( + EventTypes, Membership, JoinRules, RoomCreationPreset, +) from synapse.api.errors import StoreError, SynapseError from synapse.util import stringutils, unwrapFirstError from synapse.util.async import run_on_reactor @@ -33,6 +35,19 @@ logger = logging.getLogger(__name__) class RoomCreationHandler(BaseHandler): + PRESETS_DICT = { + RoomCreationPreset.PrivateChat: { + "join_rules": JoinRules.INVITE, + "history_visibility": "invited", + "everyone_ops": False, + }, + RoomCreationPreset.PublicChat: { + "join_rules": JoinRules.PUBLIC, + "history_visibility": "shared", + "everyone_ops": False, + }, + } + @defer.inlineCallbacks def create_room(self, user_id, room_id, config): """ Creates a new room. @@ -121,9 +136,18 @@ class RoomCreationHandler(BaseHandler): servers=[self.hs.hostname], ) + preset_config = config.get( + "preset", + RoomCreationPreset.PublicChat + if is_public + else RoomCreationPreset.PrivateChat + ) + user = UserID.from_string(user_id) creation_events = self._create_events_for_new_room( - user, room_id, is_public=is_public + user, room_id, + preset_config=preset_config, + invite_list=invite_list, ) msg_handler = self.hs.get_handlers().message_handler @@ -170,7 +194,10 @@ class RoomCreationHandler(BaseHandler): defer.returnValue(result) - def _create_events_for_new_room(self, creator, room_id, is_public=False): + def _create_events_for_new_room(self, creator, room_id, preset_config, + invite_list): + config = RoomCreationHandler.PRESETS_DICT[preset_config] + creator_id = creator.to_string() event_keys = { @@ -203,37 +230,48 @@ class RoomCreationHandler(BaseHandler): }, ) + power_level_content = { + "users": { + creator.to_string(): 100, + }, + "users_default": 0, + "events": { + EventTypes.Name: 100, + EventTypes.PowerLevels: 100, + EventTypes.RoomHistoryVisibility: 100, + }, + "events_default": 0, + "state_default": 50, + "ban": 50, + "kick": 50, + "redact": 50, + "invite": 0, + } + + if config["everyone_ops"]: + for invitee in invite_list: + power_level_content["users"][invitee] = 100 + power_levels_event = create( etype=EventTypes.PowerLevels, - content={ - "users": { - creator.to_string(): 100, - }, - "users_default": 0, - "events": { - EventTypes.Name: 100, - EventTypes.PowerLevels: 100, - EventTypes.RoomHistoryVisibility: 100, - }, - "events_default": 0, - "state_default": 50, - "ban": 50, - "kick": 50, - "redact": 50, - "invite": 0, - }, + content=power_level_content, ) - join_rule = JoinRules.PUBLIC if is_public else JoinRules.INVITE join_rules_event = create( etype=EventTypes.JoinRules, - content={"join_rule": join_rule}, + content={"join_rule": config["join_rules"]}, + ) + + history_event = create( + etype=EventTypes.RoomHistoryVisibility, + content={"history_visibility": config["history_visibility"]} ) return [ creation_event, join_event, power_levels_event, + history_event, join_rules_event, ] From 4624d6035e28c4ee05e38234f2aa1671b4ac701a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Jul 2015 10:19:07 +0100 Subject: [PATCH 133/180] Docs --- synapse/handlers/receipts.py | 11 ++++++++--- synapse/storage/receipts.py | 31 ++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 1925a4803..5b3df6932 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -149,7 +149,8 @@ class ReceiptsHandler(BaseHandler): """Gets all receipts for a room, upto the given key. """ result = yield self.store.get_linearized_receipts_for_room( - room_id, None, to_key + room_id, + to_key=to_key, ) if not result: @@ -176,7 +177,9 @@ class ReceiptEventSource(object): rooms = yield self.store.get_rooms_for_user(user.to_string()) rooms = [room.room_id for room in rooms] events = yield self.store.get_linearized_receipts_for_rooms( - rooms, from_key, to_key + rooms, + from_key=from_key, + to_key=to_key, ) defer.returnValue((events, to_key)) @@ -196,7 +199,9 @@ class ReceiptEventSource(object): rooms = yield self.store.get_rooms_for_user(user.to_string()) rooms = [room.room_id for room in rooms] events = yield self.store.get_linearized_receipts_for_rooms( - rooms, from_key, to_key + rooms, + from_key=from_key, + to_key=to_key, ) defer.returnValue((events, to_key)) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 56b9fedfd..d515a0a15 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -34,8 +34,17 @@ class ReceiptsStore(SQLBaseStore): self._receipts_stream_cache = _RoomStreamChangeCache() @defer.inlineCallbacks - def get_linearized_receipts_for_rooms(self, room_ids, from_key, to_key): + def get_linearized_receipts_for_rooms(self, room_ids, to_key, from_key=None): """Get receipts for multiple rooms for sending to clients. + + Args: + room_ids (list): List of room_ids. + to_key (int): Max stream id to fetch receipts upto. + from_key (int): Min stream id to fetch receipts from. None fetches + from the start. + + Returns: + list: A list of receipts. """ room_ids = set(room_ids) @@ -46,7 +55,9 @@ class ReceiptsStore(SQLBaseStore): results = yield defer.gatherResults( [ - self.get_linearized_receipts_for_room(room_id, from_key, to_key) + self.get_linearized_receipts_for_room( + room_id, to_key, from_key=from_key + ) for room_id in room_ids ], consumeErrors=True, @@ -55,8 +66,17 @@ class ReceiptsStore(SQLBaseStore): defer.returnValue([ev for res in results for ev in res]) @defer.inlineCallbacks - def get_linearized_receipts_for_room(self, room_id, from_key, to_key): + def get_linearized_receipts_for_room(self, room_id, to_key, from_key=None): """Get receipts for a single room for sending to clients. + + Args: + room_ids (str): The room id. + to_key (int): Max stream id to fetch receipts upto. + from_key (int): Min stream id to fetch receipts from. None fetches + from the start. + + Returns: + list: A list of receipts. """ def f(txn): if from_key: @@ -288,6 +308,9 @@ class _RoomStreamChangeCache(object): @defer.inlineCallbacks def get_rooms_changed(self, store, room_ids, key): + """Returns subset of room ids that have had new receipts since the + given key. If the key is too old it will just return the given list. + """ if key > (yield self._get_earliest_key(store)): keys = self._cache.keys() i = keys.bisect_right(key) @@ -302,6 +325,8 @@ class _RoomStreamChangeCache(object): @defer.inlineCallbacks def room_has_changed(self, store, room_id, key): + """Informs the cache that the room has been changed at the given key. + """ if key > (yield self._get_earliest_key(store)): old_key = self._room_to_key.get(room_id, None) if old_key: From b49a30a972470914531c89cd481252e414b22d0e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Jul 2015 10:20:31 +0100 Subject: [PATCH 134/180] Capitalize contants --- synapse/api/constants.py | 4 ++-- synapse/handlers/room.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 885b9c359..7156ee4e7 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -90,5 +90,5 @@ class RejectedReason(object): class RoomCreationPreset(object): - PrivateChat = "private_chat" - PublicChat = "public_chat" + PRIVATE_CHAT = "private_chat" + PUBLIC_CHAT = "public_chat" diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c65439543..11f531dc7 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -36,12 +36,12 @@ logger = logging.getLogger(__name__) class RoomCreationHandler(BaseHandler): PRESETS_DICT = { - RoomCreationPreset.PrivateChat: { + RoomCreationPreset.PRIVATE_CHAT: { "join_rules": JoinRules.INVITE, "history_visibility": "invited", "everyone_ops": False, }, - RoomCreationPreset.PublicChat: { + RoomCreationPreset.PUBLIC_CHAT: { "join_rules": JoinRules.PUBLIC, "history_visibility": "shared", "everyone_ops": False, @@ -138,9 +138,9 @@ class RoomCreationHandler(BaseHandler): preset_config = config.get( "preset", - RoomCreationPreset.PublicChat + RoomCreationPreset.PUBLIC_CHAT if is_public - else RoomCreationPreset.PrivateChat + else RoomCreationPreset.PRIVATE_CHAT ) user = UserID.from_string(user_id) From 62b4b72fe4ab084593dffe9a7fe28d51b882ccd7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 14 Jul 2015 10:33:25 +0100 Subject: [PATCH 135/180] Close, but no cigar. --- synapse/config/captcha.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index cf72dc434..15a132b4e 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -29,10 +29,10 @@ class CaptchaConfig(Config): ## Captcha ## # This Home Server's ReCAPTCHA public key. - recaptcha_private_key: "YOUR_PUBLIC_KEY" + recaptcha_private_key: "YOUR_PRIVATE_KEY" # This Home Server's ReCAPTCHA private key. - recaptcha_public_key: "YOUR_PRIVATE_KEY" + recaptcha_public_key: "YOUR_PUBLIC_KEY" # Enables ReCaptcha checks when registering, preventing signup # unless a captcha is answered. Requires a valid ReCaptcha From 002a44ac1a59f101e3df18dee5ef7c3ad87440ed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Jul 2015 10:37:42 +0100 Subject: [PATCH 136/180] s/everyone_ops/original_invitees_have_ops/ --- synapse/handlers/room.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 11f531dc7..c081efee3 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -39,12 +39,12 @@ class RoomCreationHandler(BaseHandler): RoomCreationPreset.PRIVATE_CHAT: { "join_rules": JoinRules.INVITE, "history_visibility": "invited", - "everyone_ops": False, + "original_invitees_have_ops": False, }, RoomCreationPreset.PUBLIC_CHAT: { "join_rules": JoinRules.PUBLIC, "history_visibility": "shared", - "everyone_ops": False, + "original_invitees_have_ops": False, }, } @@ -248,7 +248,7 @@ class RoomCreationHandler(BaseHandler): "invite": 0, } - if config["everyone_ops"]: + if config["original_invitees_have_ops"]: for invitee in invite_list: power_level_content["users"][invitee] = 100 From 8cedf3ce959ed1bd4d9c52a974bb097476f409f0 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 14 Jul 2015 23:53:05 +0100 Subject: [PATCH 137/180] bump up image quality a bit more as it looks crap --- synapse/rest/media/v1/thumbnailer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/media/v1/thumbnailer.py b/synapse/rest/media/v1/thumbnailer.py index 28404f2b7..1e965c363 100644 --- a/synapse/rest/media/v1/thumbnailer.py +++ b/synapse/rest/media/v1/thumbnailer.py @@ -82,7 +82,7 @@ class Thumbnailer(object): def save_image(self, output_image, output_type, output_path): output_bytes_io = BytesIO() - output_image.save(output_bytes_io, self.FORMATS[output_type], quality=70) + output_image.save(output_bytes_io, self.FORMATS[output_type], quality=80) output_bytes = output_bytes_io.getvalue() with open(output_path, "wb") as output_file: output_file.write(output_bytes) From 4da05fa0ae32425ce2755dcd479bb4c97f43b30e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 15 Jul 2015 19:28:03 +0100 Subject: [PATCH 138/180] Add back in support for remembering parameters submitted to a user-interactive auth call. --- synapse/handlers/auth.py | 6 ++++-- synapse/rest/client/v2_alpha/register.py | 11 +++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 63071653a..1ecf7fef1 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -85,8 +85,10 @@ class AuthHandler(BaseHandler): # email auth link on there). It's probably too open to abuse # because it lets unauthenticated clients store arbitrary objects # on a home server. - # sess['clientdict'] = clientdict - # self._save_session(sess) + # Revisit: Assumimg the REST APIs do sensible validation, the data + # isn't arbintrary. + sess['clientdict'] = clientdict + self._save_session(sess) pass elif 'clientdict' in sess: clientdict = sess['clientdict'] diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 72dfb876c..fa44572b7 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -57,10 +57,17 @@ class RegisterRestServlet(RestServlet): yield run_on_reactor() body = parse_request_allow_empty(request) - if 'password' not in body: - raise SynapseError(400, "", Codes.MISSING_PARAM) + # we do basic sanity checks here because the auth layerwill store these in sessions + if 'password' in body: + print "%r" % (body['password']) + if (not isinstance(body['password'], str) and + not isinstance(body['password'], unicode)) or len(body['password']) > 512: + raise SynapseError(400, "Invalid password") if 'username' in body: + if (not isinstance(body['username'], str) and + not isinstance(body['username'], unicode)) or len(body['username']) > 512: + raise SynapseError(400, "Invalid username") desired_username = body['username'] yield self.registration_handler.check_username(desired_username) From 09489499e7e4e61cd3fcf735584355ba1f780a5c Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 15 Jul 2015 19:39:18 +0100 Subject: [PATCH 139/180] pep8 + debug line --- synapse/rest/client/v2_alpha/register.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index fa44572b7..0c737d73b 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -57,16 +57,18 @@ class RegisterRestServlet(RestServlet): yield run_on_reactor() body = parse_request_allow_empty(request) - # we do basic sanity checks here because the auth layerwill store these in sessions + # we do basic sanity checks here because the auth + # layer will store these in sessions if 'password' in body: - print "%r" % (body['password']) - if (not isinstance(body['password'], str) and - not isinstance(body['password'], unicode)) or len(body['password']) > 512: + if ((not isinstance(body['password'], str) and + not isinstance(body['password'], unicode)) or + len(body['password']) > 512): raise SynapseError(400, "Invalid password") if 'username' in body: - if (not isinstance(body['username'], str) and - not isinstance(body['username'], unicode)) or len(body['username']) > 512: + if ((not isinstance(body['username'], str) and + not isinstance(body['username'], unicode)) or + len(body['username']) > 512): raise SynapseError(400, "Invalid username") desired_username = body['username'] yield self.registration_handler.check_username(desired_username) From c456d17daf1457bb341bd3c64c616d500c6c2517 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Jul 2015 15:25:29 +0100 Subject: [PATCH 140/180] Implement specifying custom initial state for /createRoom --- synapse/handlers/room.py | 100 +++++++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 40 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c081efee3..7511d294f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -27,6 +27,7 @@ from synapse.util import stringutils, unwrapFirstError from synapse.util.async import run_on_reactor from synapse.events.utils import serialize_event +from collections import OrderedDict import logging import string @@ -143,11 +144,18 @@ class RoomCreationHandler(BaseHandler): else RoomCreationPreset.PRIVATE_CHAT ) + raw_initial_state = config.get("initial_state", []) + + initial_state = OrderedDict() + for val in raw_initial_state: + initial_state[(val["type"], val.get("state_key", ""))] = val["content"] + user = UserID.from_string(user_id) creation_events = self._create_events_for_new_room( user, room_id, preset_config=preset_config, invite_list=invite_list, + initial_state=initial_state, ) msg_handler = self.hs.get_handlers().message_handler @@ -195,7 +203,7 @@ class RoomCreationHandler(BaseHandler): defer.returnValue(result) def _create_events_for_new_room(self, creator, room_id, preset_config, - invite_list): + invite_list, initial_state): config = RoomCreationHandler.PRESETS_DICT[preset_config] creator_id = creator.to_string() @@ -230,50 +238,62 @@ class RoomCreationHandler(BaseHandler): }, ) - power_level_content = { - "users": { - creator.to_string(): 100, - }, - "users_default": 0, - "events": { - EventTypes.Name: 100, - EventTypes.PowerLevels: 100, - EventTypes.RoomHistoryVisibility: 100, - }, - "events_default": 0, - "state_default": 50, - "ban": 50, - "kick": 50, - "redact": 50, - "invite": 0, - } + returned_events = [creation_event, join_event] - if config["original_invitees_have_ops"]: - for invitee in invite_list: - power_level_content["users"][invitee] = 100 + if (EventTypes.PowerLevels, '') not in initial_state: + power_level_content = { + "users": { + creator.to_string(): 100, + }, + "users_default": 0, + "events": { + EventTypes.Name: 100, + EventTypes.PowerLevels: 100, + EventTypes.RoomHistoryVisibility: 100, + }, + "events_default": 0, + "state_default": 50, + "ban": 50, + "kick": 50, + "redact": 50, + "invite": 0, + } - power_levels_event = create( - etype=EventTypes.PowerLevels, - content=power_level_content, - ) + if config["original_invitees_have_ops"]: + for invitee in invite_list: + power_level_content["users"][invitee] = 100 - join_rules_event = create( - etype=EventTypes.JoinRules, - content={"join_rule": config["join_rules"]}, - ) + power_levels_event = create( + etype=EventTypes.PowerLevels, + content=power_level_content, + ) - history_event = create( - etype=EventTypes.RoomHistoryVisibility, - content={"history_visibility": config["history_visibility"]} - ) + returned_events.append(power_levels_event) - return [ - creation_event, - join_event, - power_levels_event, - history_event, - join_rules_event, - ] + if (EventTypes.JoinRules, '') not in initial_state: + join_rules_event = create( + etype=EventTypes.JoinRules, + content={"join_rule": config["join_rules"]}, + ) + + returned_events.append(join_rules_event) + + if (EventTypes.RoomHistoryVisibility, '') not in initial_state: + history_event = create( + etype=EventTypes.RoomHistoryVisibility, + content={"history_visibility": config["history_visibility"]} + ) + + returned_events.append(history_event) + + for (etype, state_key), content in initial_state.items(): + returned_events.append(create( + etype=etype, + state_key=state_key, + content=content, + )) + + return returned_events class RoomMemberHandler(BaseHandler): From 4cab2cfa34ceb384847d8bd5dbe15147b27dc370 Mon Sep 17 00:00:00 2001 From: Matrix Date: Sat, 18 Jul 2015 19:07:12 +0100 Subject: [PATCH 141/180] Don't do any database hits in receipt handling if from_key == to_key --- synapse/handlers/receipts.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 5b3df6932..86c911c4b 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -174,6 +174,9 @@ class ReceiptEventSource(object): from_key = int(from_key) to_key = yield self.get_current_key() + if from_key == to_key: + defer.returnValue(([], to_key)) + rooms = yield self.store.get_rooms_for_user(user.to_string()) rooms = [room.room_id for room in rooms] events = yield self.store.get_linearized_receipts_for_rooms( From 4f973eb6577ba125da6faa7f307b357afe1b7f84 Mon Sep 17 00:00:00 2001 From: Matrix Date: Sat, 18 Jul 2015 19:07:33 +0100 Subject: [PATCH 142/180] Up default cache size for _RoomStreamChangeCache --- synapse/storage/receipts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index d515a0a15..7a6af98d9 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -300,7 +300,7 @@ class _RoomStreamChangeCache(object): may have changed since that key. If the key is too old then the cache will simply return all rooms. """ - def __init__(self, size_of_cache=1000): + def __init__(self, size_of_cache=10000): self._size_of_cache = size_of_cache self._room_to_key = {} self._cache = sorteddict() From b6ee0585bd0329e1841196b8e8a893630e1850d6 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 20 Jul 2015 13:55:19 +0100 Subject: [PATCH 143/180] Parse the ID given to /invite|ban|kick to make sure it looks like a user ID. --- synapse/rest/client/v1/room.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 0346afb1b..639795df2 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -412,6 +412,8 @@ class RoomMembershipRestServlet(ClientV1RestServlet): if "user_id" not in content: raise SynapseError(400, "Missing user_id key.") state_key = content["user_id"] + # make sure it looks like a user ID; it'll throw if it's invalid. + UserID.from_string(state_key); if membership_action == "kick": membership_action = "leave" From ddef5ea1267e3ec2df95b4811f1f59755a35639f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 20 Jul 2015 14:02:36 +0100 Subject: [PATCH 144/180] Remove semicolon. --- synapse/rest/client/v1/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 639795df2..b4a70cba9 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -413,7 +413,7 @@ class RoomMembershipRestServlet(ClientV1RestServlet): raise SynapseError(400, "Missing user_id key.") state_key = content["user_id"] # make sure it looks like a user ID; it'll throw if it's invalid. - UserID.from_string(state_key); + UserID.from_string(state_key) if membership_action == "kick": membership_action = "leave" From 53d1174aa9f984b2c2b012e9acd1bf52512998f9 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 20 Jul 2015 06:32:12 -0700 Subject: [PATCH 145/180] Improve naming --- synapse/notifier.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 85ae34313..dbd8efe9f 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -352,10 +352,10 @@ class Notifier(object): after_id = getattr(after_token, keyname) if before_id == after_id: continue - stuff, new_key = yield source.get_new_events_for_user( + new_events, new_key = yield source.get_new_events_for_user( user, getattr(from_token, keyname), limit, ) - events.extend(stuff) + events.extend(new_events) end_token = end_token.copy_and_replace(keyname, new_key) if events: From 3b5823c74d5bffc68068284145cc78a33476ac84 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Jul 2015 13:08:33 +0100 Subject: [PATCH 146/180] s/take/claim/ for end to end key APIs --- synapse/rest/client/v2_alpha/keys.py | 10 +++++----- synapse/storage/end_to_end_keys.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/v2_alpha/keys.py index f03126775..9a0c84228 100644 --- a/synapse/rest/client/v2_alpha/keys.py +++ b/synapse/rest/client/v2_alpha/keys.py @@ -207,9 +207,9 @@ class KeyQueryServlet(RestServlet): class OneTimeKeyServlet(RestServlet): """ - GET /keys/take/// HTTP/1.1 + GET /keys/claim/// HTTP/1.1 - POST /keys/take HTTP/1.1 + POST /keys/claim HTTP/1.1 { "one_time_keys": { "": { @@ -226,7 +226,7 @@ class OneTimeKeyServlet(RestServlet): """ PATTERN = client_v2_pattern( - "/keys/take(?:/?|(?:/" + "/keys/claim(?:/?|(?:/" "(?P[^/]*)/(?P[^/]*)/(?P[^/]*)" ")?)" ) @@ -240,7 +240,7 @@ class OneTimeKeyServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id, device_id, algorithm): yield self.auth.get_user_by_req(request) - results = yield self.store.take_e2e_one_time_keys( + results = yield self.store.claim_e2e_one_time_keys( [(user_id, device_id, algorithm)] ) defer.returnValue(self.json_result(request, results)) @@ -256,7 +256,7 @@ class OneTimeKeyServlet(RestServlet): for user_id, device_keys in body.get("one_time_keys", {}).items(): for device_id, algorithm in device_keys.items(): query.append((user_id, device_id, algorithm)) - results = yield self.store.take_e2e_one_time_keys(query) + results = yield self.store.claim_e2e_one_time_keys(query) defer.returnValue(self.json_result(request, results)) def json_result(self, request, results): diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index 99dc864e4..325740d7d 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -95,9 +95,9 @@ class EndToEndKeyStore(SQLBaseStore): "count_e2e_one_time_keys", _count_e2e_one_time_keys ) - def take_e2e_one_time_keys(self, query_list): + def claim_e2e_one_time_keys(self, query_list): """Take a list of one time keys out of the database""" - def _take_e2e_one_time_keys(txn): + def _claim_e2e_one_time_keys(txn): sql = ( "SELECT key_id, key_json FROM e2e_one_time_keys_json" " WHERE user_id = ? AND device_id = ? AND algorithm = ?" @@ -121,5 +121,5 @@ class EndToEndKeyStore(SQLBaseStore): txn.execute(sql, (user_id, device_id, algorithm, key_id)) return result return self.runInteraction( - "take_e2e_one_time_keys", _take_e2e_one_time_keys + "claim_e2e_one_time_keys", _claim_e2e_one_time_keys ) From cf7a40b08a381ee5715f915effd63dfe241a8d61 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 21 Jul 2015 16:08:00 -0700 Subject: [PATCH 147/180] I think this was what was intended... --- synapse/storage/end_to_end_keys.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index 325740d7d..69287c43d 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -52,6 +52,7 @@ class EndToEndKeyStore(SQLBaseStore): ) for row in rows: user_result[row["device_id"]] = row["key_json"] + result[user_id] = user_result return result return self.runInteraction("get_e2e_device_keys", _get_e2e_device_keys) From 20c0324e9c0ebe569d30bd6541bc8b5c9d3c7ae2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 21 Jul 2015 16:21:37 -0700 Subject: [PATCH 148/180] Dodesn't seem to make any difference: guess it does work with the object reference --- synapse/storage/end_to_end_keys.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index 69287c43d..325740d7d 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -52,7 +52,6 @@ class EndToEndKeyStore(SQLBaseStore): ) for row in rows: user_result[row["device_id"]] = row["key_json"] - result[user_id] = user_result return result return self.runInteraction("get_e2e_device_keys", _get_e2e_device_keys) From a56eccbbfc377d071a403fde875e02cf8fb1b68a Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 21 Jul 2015 16:38:16 -0700 Subject: [PATCH 149/180] Query for all the ones we were asked about, not just the last... --- synapse/rest/client/v2_alpha/keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/keys.py b/synapse/rest/client/v2_alpha/keys.py index 9a0c84228..5f3a6207b 100644 --- a/synapse/rest/client/v2_alpha/keys.py +++ b/synapse/rest/client/v2_alpha/keys.py @@ -180,7 +180,7 @@ class KeyQueryServlet(RestServlet): else: for device_id in device_ids: query.append((user_id, device_id)) - results = yield self.store.get_e2e_device_keys([(user_id, device_id)]) + results = yield self.store.get_e2e_device_keys(query) defer.returnValue(self.json_result(request, results)) @defer.inlineCallbacks From 4e2e67fd506899e0c64b5fdfcac91e660de4260b Mon Sep 17 00:00:00 2001 From: Matrix Date: Wed, 22 Jul 2015 16:13:46 +0100 Subject: [PATCH 150/180] Disable receipts for now --- synapse/handlers/receipts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 86c911c4b..415dd339f 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -171,6 +171,7 @@ class ReceiptEventSource(object): @defer.inlineCallbacks def get_new_events_for_user(self, user, from_key, limit): + defer.returnValue(([], from_key)) from_key = int(from_key) to_key = yield self.get_current_key() @@ -193,6 +194,7 @@ class ReceiptEventSource(object): @defer.inlineCallbacks def get_pagination_rows(self, user, config, key): to_key = int(config.from_key) + defer.returnValue(([], to_key)) if config.to_key: from_key = int(config.to_key) From 103e1c2431e92959c8b265335714c1a2c5d5dd70 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 23 Jul 2015 11:12:49 +0100 Subject: [PATCH 151/180] Pick larger than desired thumbnail for 'crop' --- synapse/rest/media/v1/thumbnail_resource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 4a9b6d8ee..61f88e486 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -162,11 +162,12 @@ class ThumbnailResource(BaseMediaResource): t_method = info["thumbnail_method"] if t_method == "scale" or t_method == "crop": aspect_quality = abs(d_w * t_h - d_h * t_w) + min_quality = 0 if d_w <= t_w and d_h <= t_h else 1 size_quality = abs((d_w - t_w) * (d_h - t_h)) type_quality = desired_type != info["thumbnail_type"] length_quality = info["thumbnail_length"] info_list.append(( - aspect_quality, size_quality, type_quality, + aspect_quality, min_quality, size_quality, type_quality, length_quality, info )) if info_list: From 2b4f47db9c13840c4b9dbbffb28d7860fe007d68 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 23 Jul 2015 14:52:29 +0100 Subject: [PATCH 152/180] Generate local thumbnails on a thread --- synapse/rest/media/v1/base_resource.py | 77 ++++++++++++++------------ 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index c43ae0314..84e1961a2 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -244,43 +244,52 @@ class BaseMediaResource(Resource): ) return - scales = set() - crops = set() - for r_width, r_height, r_method, r_type in requirements: - if r_method == "scale": - t_width, t_height = thumbnailer.aspect(r_width, r_height) - scales.add(( - min(m_width, t_width), min(m_height, t_height), r_type, + local_thumbnails = [] + + def generate_thumbnails(): + scales = set() + crops = set() + for r_width, r_height, r_method, r_type in requirements: + if r_method == "scale": + t_width, t_height = thumbnailer.aspect(r_width, r_height) + scales.add(( + min(m_width, t_width), min(m_height, t_height), r_type, + )) + elif r_method == "crop": + crops.add((r_width, r_height, r_type)) + + for t_width, t_height, t_type in scales: + t_method = "scale" + t_path = self.filepaths.local_media_thumbnail( + media_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) + + local_thumbnails.append(( + media_id, t_width, t_height, t_type, t_method, t_len )) - elif r_method == "crop": - crops.add((r_width, r_height, r_type)) - for t_width, t_height, t_type in scales: - t_method = "scale" - t_path = self.filepaths.local_media_thumbnail( - media_id, t_width, t_height, t_type, t_method - ) - self._makedirs(t_path) - t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) - yield self.store.store_local_thumbnail( - media_id, t_width, t_height, t_type, t_method, t_len - ) + for t_width, t_height, t_type in crops: + if (t_width, t_height, t_type) in scales: + # If the aspect ratio of the cropped thumbnail matches a purely + # scaled one then there is no point in calculating a separate + # thumbnail. + continue + t_method = "crop" + t_path = self.filepaths.local_media_thumbnail( + media_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) + local_thumbnails.append(( + media_id, t_width, t_height, t_type, t_method, t_len + )) - for t_width, t_height, t_type in crops: - if (t_width, t_height, t_type) in scales: - # If the aspect ratio of the cropped thumbnail matches a purely - # scaled one then there is no point in calculating a separate - # thumbnail. - continue - t_method = "crop" - t_path = self.filepaths.local_media_thumbnail( - media_id, t_width, t_height, t_type, t_method - ) - self._makedirs(t_path) - t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) - yield self.store.store_local_thumbnail( - media_id, t_width, t_height, t_type, t_method, t_len - ) + yield threads.deferToThread(generate_thumbnails) + + for l in local_thumbnails: + yield self.store.store_local_thumbnail(*l) defer.returnValue({ "width": m_width, From 39e21ea51cf94f436f1f845f24c1b04f11b92c6f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 27 Jul 2015 13:57:29 +0100 Subject: [PATCH 153/180] Add support for using keyword arguments with cached functions --- synapse/storage/_base.py | 40 ++++++++++++++++++++++++++++++------ synapse/storage/keys.py | 5 ++--- synapse/storage/push_rule.py | 8 +++----- synapse/storage/receipts.py | 5 ++--- synapse/storage/room.py | 5 ++--- synapse/storage/state.py | 5 ++--- 6 files changed, 45 insertions(+), 23 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 8f812f0fd..f1265541b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -27,6 +27,7 @@ from twisted.internet import defer from collections import namedtuple, OrderedDict import functools +import inspect import sys import time import threading @@ -141,13 +142,28 @@ class CacheDescriptor(object): which can be used to insert values into the cache specifically, without calling the calculation function. """ - def __init__(self, orig, max_entries=1000, num_args=1, lru=False): + def __init__(self, orig, max_entries=1000, num_args=1, lru=False, + inlineCallbacks=False): self.orig = orig + if inlineCallbacks: + self.function_to_call = defer.inlineCallbacks(orig) + else: + self.function_to_call = orig + self.max_entries = max_entries self.num_args = num_args self.lru = lru + self.arg_names = inspect.getargspec(orig).args[1:num_args+1] + + if len(self.arg_names) < self.num_args: + raise Exception( + "Not enough explicit positional arguments to key off of for %r." + " (@cached cannot key off of *args or **kwars)" + % (orig.__name__,) + ) + def __get__(self, obj, objtype=None): cache = Cache( name=self.orig.__name__, @@ -158,11 +174,13 @@ class CacheDescriptor(object): @functools.wraps(self.orig) @defer.inlineCallbacks - def wrapped(*keyargs): + def wrapped(*args, **kwargs): + arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs) + keyargs = [arg_dict[arg_nm] for arg_nm in self.arg_names] try: - cached_result = cache.get(*keyargs[:self.num_args]) + cached_result = cache.get(*keyargs) if DEBUG_CACHES: - actual_result = yield self.orig(obj, *keyargs) + actual_result = yield self.function_to_call(obj, *args, **kwargs) if actual_result != cached_result: logger.error( "Stale cache entry %s%r: cached: %r, actual %r", @@ -177,9 +195,9 @@ class CacheDescriptor(object): # while the SELECT is executing (SYN-369) sequence = cache.sequence - ret = yield self.orig(obj, *keyargs) + ret = yield self.function_to_call(obj, *args, **kwargs) - cache.update(sequence, *keyargs[:self.num_args] + (ret,)) + cache.update(sequence, *(keyargs + [ret])) defer.returnValue(ret) @@ -201,6 +219,16 @@ def cached(max_entries=1000, num_args=1, lru=False): ) +def cachedInlineCallbacks(max_entries=1000, num_args=1, lru=False): + return lambda orig: CacheDescriptor( + orig, + max_entries=max_entries, + num_args=num_args, + lru=lru, + inlineCallbacks=True, + ) + + class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging and metrics to the .execute() diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 940a5f7e0..e3f98f0cd 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from _base import SQLBaseStore, cached +from _base import SQLBaseStore, cachedInlineCallbacks from twisted.internet import defer @@ -71,8 +71,7 @@ class KeyStore(SQLBaseStore): desc="store_server_certificate", ) - @cached() - @defer.inlineCallbacks + @cachedInlineCallbacks() def get_all_server_verify_keys(self, server_name): rows = yield self._simple_select_list( table="server_signature_keys", diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 4cac118d1..a220f3632 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore, cached +from ._base import SQLBaseStore, cachedInlineCallbacks from twisted.internet import defer import logging @@ -23,8 +23,7 @@ logger = logging.getLogger(__name__) class PushRuleStore(SQLBaseStore): - @cached() - @defer.inlineCallbacks + @cachedInlineCallbacks() def get_push_rules_for_user(self, user_name): rows = yield self._simple_select_list( table=PushRuleTable.table_name, @@ -41,8 +40,7 @@ class PushRuleStore(SQLBaseStore): defer.returnValue(rows) - @cached() - @defer.inlineCallbacks + @cachedInlineCallbacks() def get_push_rules_enabled_for_user(self, user_name): results = yield self._simple_select_list( table=PushRuleEnableTable.table_name, diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py index 7a6af98d9..b79d6683c 100644 --- a/synapse/storage/receipts.py +++ b/synapse/storage/receipts.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore, cached +from ._base import SQLBaseStore, cachedInlineCallbacks from twisted.internet import defer @@ -128,8 +128,7 @@ class ReceiptsStore(SQLBaseStore): def get_max_receipt_stream_id(self): return self._receipts_id_gen.get_max_token(self) - @cached - @defer.inlineCallbacks + @cachedInlineCallbacks() def get_graph_receipts_for_room(self, room_id): """Get receipts for sending to remote servers. """ diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 4612a8aa8..dd5bc2c8f 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -17,7 +17,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError -from ._base import SQLBaseStore, cached +from ._base import SQLBaseStore, cachedInlineCallbacks import collections import logging @@ -186,8 +186,7 @@ class RoomStore(SQLBaseStore): } ) - @cached() - @defer.inlineCallbacks + @cachedInlineCallbacks() def get_room_name_and_aliases(self, room_id): def f(txn): sql = ( diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 47bec6549..55c6d5289 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore, cached +from ._base import SQLBaseStore, cached, cachedInlineCallbacks from twisted.internet import defer @@ -189,8 +189,7 @@ class StateStore(SQLBaseStore): events = yield self._get_events(event_ids, get_prev_content=False) defer.returnValue(events) - @cached(num_args=3) - @defer.inlineCallbacks + @cachedInlineCallbacks(num_args=3) def get_current_state_for_key(self, room_id, event_type, state_key): def f(txn): sql = ( From a4d62ba36afc54d4e60f1371fe9b31e8b8e6834c Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 28 Jul 2015 17:34:12 +0100 Subject: [PATCH 154/180] Fix v2_alpha registration. Add unit tests. V2 Registration forced everyone (including ASes) to create a password for a user, when ASes should be able to omit passwords. Also unbreak AS registration in general which checked too early if the given username was claimed by an AS; it was checked before knowing if the AS was the one doing the registration! Add unit tests for AS reg, user reg and disabled_registration flag. --- synapse/handlers/register.py | 3 +- synapse/rest/client/v2_alpha/register.py | 124 ++++++++++-------- tests/rest/client/v2_alpha/test_register.py | 132 ++++++++++++++++++++ 3 files changed, 204 insertions(+), 55 deletions(-) create mode 100644 tests/rest/client/v2_alpha/test_register.py diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index a1288b425..f81d75017 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -73,7 +73,8 @@ class RegistrationHandler(BaseHandler): localpart : The local part of the user ID to register. If None, one will be randomly generated. password (str) : The password to assign to this user so they can - login again. + login again. This can be None which means they cannot login again + via a password (e.g. the user is an application service user). Returns: A tuple of (user_id, access_token). Raises: diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 0c737d73b..e1c42dd51 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -19,7 +19,7 @@ from synapse.api.constants import LoginType from synapse.api.errors import SynapseError, Codes from synapse.http.servlet import RestServlet -from ._base import client_v2_pattern, parse_request_allow_empty +from ._base import client_v2_pattern, parse_json_dict_from_request import logging import hmac @@ -55,30 +55,52 @@ class RegisterRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request): yield run_on_reactor() + body = parse_json_dict_from_request(request) - body = parse_request_allow_empty(request) - # we do basic sanity checks here because the auth - # layer will store these in sessions + # we do basic sanity checks here because the auth layer will store these + # in sessions. Pull out the username/password provided to us. + desired_password = None if 'password' in body: - if ((not isinstance(body['password'], str) and - not isinstance(body['password'], unicode)) or + if (not isinstance(body['password'], basestring) or len(body['password']) > 512): raise SynapseError(400, "Invalid password") + desired_password = body["password"] + desired_username = None if 'username' in body: - if ((not isinstance(body['username'], str) and - not isinstance(body['username'], unicode)) or + if (not isinstance(body['username'], basestring) or len(body['username']) > 512): raise SynapseError(400, "Invalid username") desired_username = body['username'] - yield self.registration_handler.check_username(desired_username) - is_using_shared_secret = False - is_application_server = False - - service = None + appservice = None if 'access_token' in request.args: - service = yield self.auth.get_appservice_by_req(request) + appservice = yield self.auth.get_appservice_by_req(request) + + # fork off as soon as possible for ASes and shared secret auth which + # have completely different registration flows to normal users + + # == Application Service Registration == + if appservice: + result = yield self._do_appservice_registration(desired_username) + defer.returnValue((200, result)) # we throw for non 200 responses + return + + # == Shared Secret Registration == (e.g. create new user scripts) + if 'mac' in body: + # FIXME: Should we really be determining if this is shared secret + # auth based purely on the 'mac' key? + result = yield self._do_shared_secret_registration( + desired_username, desired_password, body["mac"] + ) + defer.returnValue((200, result)) # we throw for non 200 responses + return + + # == Normal User Registration == (everyone else) + if self.hs.config.disable_registration: + raise SynapseError(403, "Registration has been disabled") + + yield self.registration_handler.check_username(desired_username) if self.hs.config.enable_registration_captcha: flows = [ @@ -91,39 +113,20 @@ class RegisterRestServlet(RestServlet): [LoginType.EMAIL_IDENTITY] ] - result = None - if service: - is_application_server = True - params = body - elif 'mac' in body: - # Check registration-specific shared secret auth - if 'username' not in body: - raise SynapseError(400, "", Codes.MISSING_PARAM) - self._check_shared_secret_auth( - body['username'], body['mac'] - ) - is_using_shared_secret = True - params = body - else: - authed, result, params = yield self.auth_handler.check_auth( - flows, body, self.hs.get_ip_from_request(request) - ) - - if not authed: - defer.returnValue((401, result)) - - can_register = ( - not self.hs.config.disable_registration - or is_application_server - or is_using_shared_secret + authed, result, params = yield self.auth_handler.check_auth( + flows, body, self.hs.get_ip_from_request(request) ) - if not can_register: - raise SynapseError(403, "Registration has been disabled") + if not authed: + defer.returnValue((401, result)) + return + + # NB: This may be from the auth handler and NOT from the POST if 'password' not in params: - raise SynapseError(400, "", Codes.MISSING_PARAM) - desired_username = params['username'] if 'username' in params else None - new_password = params['password'] + raise SynapseError(400, "Missing password.", Codes.MISSING_PARAM) + + desired_username = params.get("username", None) + new_password = params.get("password", None) (user_id, token) = yield self.registration_handler.register( localpart=desired_username, @@ -156,18 +159,21 @@ class RegisterRestServlet(RestServlet): else: logger.info("bind_email not specified: not binding email") - result = { - "user_id": user_id, - "access_token": token, - "home_server": self.hs.hostname, - } - + result = self._create_registration_details(user_id, token) defer.returnValue((200, result)) def on_OPTIONS(self, _): return 200, {} - def _check_shared_secret_auth(self, username, mac): + @defer.inlineCallbacks + def _do_appservice_registration(self, username): + (user_id, token) = yield self.registration_handler.register( + localpart=username + ) + defer.returnValue(self._create_registration_details(user_id, token)) + + @defer.inlineCallbacks + def _do_shared_secret_registration(self, username, password, mac): if not self.hs.config.registration_shared_secret: raise SynapseError(400, "Shared secret registration is not enabled") @@ -183,13 +189,23 @@ class RegisterRestServlet(RestServlet): digestmod=sha1, ).hexdigest() - if compare_digest(want_mac, got_mac): - return True - else: + if not compare_digest(want_mac, got_mac): raise SynapseError( 403, "HMAC incorrect", ) + (user_id, token) = yield self.registration_handler.register( + localpart=username, password=password + ) + defer.returnValue(self._create_registration_details(user_id, token)) + + def _create_registration_details(self, user_id, token): + return { + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname, + } + def register_servlets(hs, http_server): RegisterRestServlet(hs).register(http_server) diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py new file mode 100644 index 000000000..3edc2ec2e --- /dev/null +++ b/tests/rest/client/v2_alpha/test_register.py @@ -0,0 +1,132 @@ +from synapse.rest.client.v2_alpha.register import RegisterRestServlet +from synapse.api.errors import SynapseError +from twisted.internet import defer +from mock import Mock, MagicMock +from tests import unittest +import json + + +class RegisterRestServletTestCase(unittest.TestCase): + + def setUp(self): + # do the dance to hook up request data to self.request_data + self.request_data = "" + self.request = Mock( + content=Mock(read=Mock(side_effect=lambda: self.request_data)), + ) + self.request.args = {} + + self.appservice = None + self.auth = Mock(get_appservice_by_req=Mock( + side_effect=lambda x: defer.succeed(self.appservice)) + ) + + self.auth_result = (False, None, None) + self.auth_handler = Mock( + check_auth=Mock(side_effect=lambda x,y,z: self.auth_result) + ) + self.registration_handler = Mock() + self.identity_handler = Mock() + self.login_handler = Mock() + + # do the dance to hook it up to the hs global + self.handlers = Mock( + auth_handler=self.auth_handler, + registration_handler=self.registration_handler, + identity_handler=self.identity_handler, + login_handler=self.login_handler + ) + self.hs = Mock() + self.hs.hostname = "superbig~testing~thing.com" + self.hs.get_auth = Mock(return_value=self.auth) + self.hs.get_handlers = Mock(return_value=self.handlers) + self.hs.config.disable_registration = False + + # init the thing we're testing + self.servlet = RegisterRestServlet(self.hs) + + @defer.inlineCallbacks + def test_POST_appservice_registration_valid(self): + user_id = "@kermit:muppet" + token = "kermits_access_token" + self.request.args = { + "access_token": "i_am_an_app_service" + } + self.request_data = json.dumps({ + "username": "kermit" + }) + self.appservice = { + "id": "1234" + } + self.registration_handler.register = Mock(return_value=(user_id, token)) + result = yield self.servlet.on_POST(self.request) + self.assertEquals(result, (200, { + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname + })) + + @defer.inlineCallbacks + def test_POST_appservice_registration_invalid(self): + self.request.args = { + "access_token": "i_am_an_app_service" + } + self.request_data = json.dumps({ + "username": "kermit" + }) + self.appservice = None # no application service exists + result = yield self.servlet.on_POST(self.request) + self.assertEquals(result, (401, None)) + + def test_POST_bad_password(self): + self.request_data = json.dumps({ + "username": "kermit", + "password": 666 + }) + d = self.servlet.on_POST(self.request) + return self.assertFailure(d, SynapseError) + + def test_POST_bad_username(self): + self.request_data = json.dumps({ + "username": 777, + "password": "monkey" + }) + d = self.servlet.on_POST(self.request) + return self.assertFailure(d, SynapseError) + + @defer.inlineCallbacks + def test_POST_user_valid(self): + user_id = "@kermit:muppet" + token = "kermits_access_token" + self.request_data = json.dumps({ + "username": "kermit", + "password": "monkey" + }) + self.registration_handler.check_username = Mock(return_value=True) + self.auth_result = (True, None, { + "username": "kermit", + "password": "monkey" + }) + self.registration_handler.register = Mock(return_value=(user_id, token)) + + result = yield self.servlet.on_POST(self.request) + self.assertEquals(result, (200, { + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname + })) + + def test_POST_disabled_registration(self): + self.hs.config.disable_registration = True + self.request_data = json.dumps({ + "username": "kermit", + "password": "monkey" + }) + self.registration_handler.check_username = Mock(return_value=True) + self.auth_result = (True, None, { + "username": "kermit", + "password": "monkey" + }) + self.registration_handler.register = Mock(return_value=("@user:id", "t")) + d = self.servlet.on_POST(self.request) + return self.assertFailure(d, SynapseError) \ No newline at end of file From c772dffc9f024ce3135039c9f8032f09cc730841 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 29 Jul 2015 09:39:55 +0100 Subject: [PATCH 155/180] improve OS X instructions and remove all the leading $'s to make it easier to c+p commands --- README.rst | 92 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/README.rst b/README.rst index 21e347968..5ff53f2df 100644 --- a/README.rst +++ b/README.rst @@ -101,25 +101,26 @@ header files for python C extensions. Installing prerequisites on Ubuntu or Debian:: - $ sudo apt-get install build-essential python2.7-dev libffi-dev \ - python-pip python-setuptools sqlite3 \ - libssl-dev python-virtualenv libjpeg-dev + sudo apt-get install build-essential python2.7-dev libffi-dev \ + python-pip python-setuptools sqlite3 \ + libssl-dev python-virtualenv libjpeg-dev Installing prerequisites on ArchLinux:: - $ sudo pacman -S base-devel python2 python-pip \ - python-setuptools python-virtualenv sqlite3 + sudo pacman -S base-devel python2 python-pip \ + python-setuptools python-virtualenv sqlite3 Installing prerequisites on Mac OS X:: - $ xcode-select --install - $ sudo pip install virtualenv + xcode-select --install + sudo easy_install pip + sudo pip install virtualenv To install the synapse homeserver run:: - $ virtualenv -p python2.7 ~/.synapse - $ source ~/.synapse/bin/activate - $ pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master + virtualenv -p python2.7 ~/.synapse + source ~/.synapse/bin/activate + pip install --process-dependency-links https://github.com/matrix-org/synapse/tarball/master This installs synapse, along with the libraries it uses, into a virtual environment under ``~/.synapse``. Feel free to pick a different directory @@ -132,8 +133,8 @@ above in Docker at https://registry.hub.docker.com/u/silviof/docker-matrix/. To set up your homeserver, run (in your virtualenv, as before):: - $ cd ~/.synapse - $ python -m synapse.app.homeserver \ + cd ~/.synapse + python -m synapse.app.homeserver \ --server-name machine.my.domain.name \ --config-path homeserver.yaml \ --generate-config @@ -192,9 +193,9 @@ Running Synapse To actually run your new homeserver, pick a working directory for Synapse to run (e.g. ``~/.synapse``), and:: - $ cd ~/.synapse - $ source ./bin/activate - $ synctl start + cd ~/.synapse + source ./bin/activate + synctl start Platform Specific Instructions ============================== @@ -212,12 +213,12 @@ defaults to python 3, but synapse currently assumes python 2.7 by default: pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):: - $ sudo pip2.7 install --upgrade pip + sudo pip2.7 install --upgrade pip You also may need to explicitly specify python 2.7 again during the install request:: - $ pip2.7 install --process-dependency-links \ + pip2.7 install --process-dependency-links \ https://github.com/matrix-org/synapse/tarball/master If you encounter an error with lib bcrypt causing an Wrong ELF Class: @@ -225,13 +226,13 @@ ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly compile it under the right architecture. (This should not be needed if installing under virtualenv):: - $ sudo pip2.7 uninstall py-bcrypt - $ sudo pip2.7 install py-bcrypt + sudo pip2.7 uninstall py-bcrypt + sudo pip2.7 install py-bcrypt During setup of Synapse you need to call python2.7 directly again:: - $ cd ~/.synapse - $ python2.7 -m synapse.app.homeserver \ + cd ~/.synapse + python2.7 -m synapse.app.homeserver \ --server-name machine.my.domain.name \ --config-path homeserver.yaml \ --generate-config @@ -279,22 +280,22 @@ Synapse requires pip 1.7 or later, so if your OS provides too old a version and you get errors about ``error: no such option: --process-dependency-links`` you may need to manually upgrade it:: - $ sudo pip install --upgrade pip + sudo pip install --upgrade pip If pip crashes mid-installation for reason (e.g. lost terminal), pip may refuse to run until you remove the temporary installation directory it created. To reset the installation:: - $ rm -rf /tmp/pip_install_matrix + rm -rf /tmp/pip_install_matrix pip seems to leak *lots* of memory during installation. For instance, a Linux host with 512MB of RAM may run out of memory whilst installing Twisted. If this happens, you will have to individually install the dependencies which are failing, e.g.:: - $ pip install twisted + pip install twisted -On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you +On OS X, if you encounter clang: error: unknown argument: '-mno-fused-madd' you will need to export CFLAGS=-Qunused-arguments. Troubleshooting Running @@ -310,10 +311,11 @@ correctly, causing all tests to fail with errors about missing "sodium.h". To fix try re-installing from PyPI or directly from (https://github.com/pyca/pynacl):: - $ # Install from PyPI - $ pip install --user --upgrade --force pynacl - $ # Install from github - $ pip install --user https://github.com/pyca/pynacl/tarball/master + # Install from PyPI + pip install --user --upgrade --force pynacl + + # Install from github + pip install --user https://github.com/pyca/pynacl/tarball/master ArchLinux ~~~~~~~~~ @@ -321,7 +323,7 @@ ArchLinux If running `$ synctl start` fails with 'returned non-zero exit status 1', you will need to explicitly call Python2.7 - either running as:: - $ python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml + python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml ...or by editing synctl with the correct python executable. @@ -331,16 +333,16 @@ Synapse Development To check out a synapse for development, clone the git repo into a working directory of your choice:: - $ git clone https://github.com/matrix-org/synapse.git - $ cd synapse + git clone https://github.com/matrix-org/synapse.git + cd synapse Synapse has a number of external dependencies, that are easiest to install using pip and a virtualenv:: - $ virtualenv env - $ source env/bin/activate - $ python synapse/python_dependencies.py | xargs -n1 pip install - $ pip install setuptools_trial mock + virtualenv env + source env/bin/activate + python synapse/python_dependencies.py | xargs -n1 pip install + pip install setuptools_trial mock This will run a process of downloading and installing all the needed dependencies into a virtual env. @@ -348,7 +350,7 @@ dependencies into a virtual env. Once this is done, you may wish to run Synapse's unit tests, to check that everything is installed as it should be:: - $ python setup.py test + python setup.py test This should end with a 'PASSED' result:: @@ -389,11 +391,11 @@ IDs: For the first form, simply pass the required hostname (of the machine) as the --server-name parameter:: - $ python -m synapse.app.homeserver \ + python -m synapse.app.homeserver \ --server-name machine.my.domain.name \ --config-path homeserver.yaml \ --generate-config - $ python -m synapse.app.homeserver --config-path homeserver.yaml + python -m synapse.app.homeserver --config-path homeserver.yaml Alternatively, you can run ``synctl start`` to guide you through the process. @@ -410,11 +412,11 @@ record would then look something like:: At this point, you should then run the homeserver with the hostname of this SRV record, as that is the name other machines will expect it to have:: - $ python -m synapse.app.homeserver \ + python -m synapse.app.homeserver \ --server-name YOURDOMAIN \ --config-path homeserver.yaml \ --generate-config - $ python -m synapse.app.homeserver --config-path homeserver.yaml + python -m synapse.app.homeserver --config-path homeserver.yaml You may additionally want to pass one or more "-v" options, in order to @@ -428,7 +430,7 @@ private federation (``localhost:8080``, ``localhost:8081`` and ``localhost:8082``) which you can then access through the webclient running at http://localhost:8080. Simply run:: - $ demo/start.sh + demo/start.sh This is mainly useful just for development purposes. @@ -502,10 +504,10 @@ Building Internal API Documentation Before building internal API documentation install sphinx and sphinxcontrib-napoleon:: - $ pip install sphinx - $ pip install sphinxcontrib-napoleon + pip install sphinx + pip install sphinxcontrib-napoleon Building internal API documentation:: - $ python setup.py build_sphinx + python setup.py build_sphinx From 11b0a3407485e98082bf06d771e5ae2f68106ca7 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 29 Jul 2015 10:00:54 +0100 Subject: [PATCH 156/180] Use the same reg paths as register v1 for ASes. Namely this means using registration_handler.appservice_register. --- synapse/rest/client/v2_alpha/register.py | 10 ++++++---- tests/rest/client/v2_alpha/test_register.py | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index e1c42dd51..cf54e1dac 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -82,7 +82,9 @@ class RegisterRestServlet(RestServlet): # == Application Service Registration == if appservice: - result = yield self._do_appservice_registration(desired_username) + result = yield self._do_appservice_registration( + desired_username, request.args["access_token"][0] + ) defer.returnValue((200, result)) # we throw for non 200 responses return @@ -166,9 +168,9 @@ class RegisterRestServlet(RestServlet): return 200, {} @defer.inlineCallbacks - def _do_appservice_registration(self, username): - (user_id, token) = yield self.registration_handler.register( - localpart=username + def _do_appservice_registration(self, username, as_token): + (user_id, token) = yield self.registration_handler.appservice_register( + username, as_token ) defer.returnValue(self._create_registration_details(user_id, token)) diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py index 3edc2ec2e..66fd25964 100644 --- a/tests/rest/client/v2_alpha/test_register.py +++ b/tests/rest/client/v2_alpha/test_register.py @@ -58,7 +58,9 @@ class RegisterRestServletTestCase(unittest.TestCase): self.appservice = { "id": "1234" } - self.registration_handler.register = Mock(return_value=(user_id, token)) + self.registration_handler.appservice_register = Mock( + return_value=(user_id, token) + ) result = yield self.servlet.on_POST(self.request) self.assertEquals(result, (200, { "user_id": user_id, From 28d07a02e42c90732b708de870af2f9b21ba13f0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 3 Aug 2015 15:31:21 +0100 Subject: [PATCH 157/180] Add vector.im as trusted ID server --- synapse/handlers/identity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 6200e1077..c1095708a 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -44,7 +44,7 @@ class IdentityHandler(BaseHandler): http_client = SimpleHttpClient(self.hs) # XXX: make this configurable! # trustedIdServers = ['matrix.org', 'localhost:8090'] - trustedIdServers = ['matrix.org'] + trustedIdServers = ['matrix.org', 'vector.im'] if 'id_server' in creds: id_server = creds['id_server'] From 7148aaf5d0f75c463c93ac69885d05160fee4d4a Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 3 Aug 2015 17:03:27 +0100 Subject: [PATCH 158/180] Don't try & check the username if we don't have one (which we won't if it's been saved in the auth layer) --- synapse/rest/client/v2_alpha/register.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index cf54e1dac..b5926f9ca 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -102,7 +102,8 @@ class RegisterRestServlet(RestServlet): if self.hs.config.disable_registration: raise SynapseError(403, "Registration has been disabled") - yield self.registration_handler.check_username(desired_username) + if desired_username is not None: + yield self.registration_handler.check_username(desired_username) if self.hs.config.enable_registration_captcha: flows = [ From a89559d79714b8dd07ec5a73f2629d003fcad92c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Aug 2015 15:39:47 +0100 Subject: [PATCH 159/180] Use LRU cache by default --- synapse/storage/_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 8f812f0fd..2a601e37e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -57,7 +57,7 @@ cache_counter = metrics.register_cache( class Cache(object): - def __init__(self, name, max_entries=1000, keylen=1, lru=False): + def __init__(self, name, max_entries=1000, keylen=1, lru=True): if lru: self.cache = LruCache(max_size=max_entries) self.max_entries = None @@ -141,7 +141,7 @@ class CacheDescriptor(object): which can be used to insert values into the cache specifically, without calling the calculation function. """ - def __init__(self, orig, max_entries=1000, num_args=1, lru=False): + def __init__(self, orig, max_entries=1000, num_args=1, lru=True): self.orig = orig self.max_entries = max_entries @@ -192,7 +192,7 @@ class CacheDescriptor(object): return wrapped -def cached(max_entries=1000, num_args=1, lru=False): +def cached(max_entries=1000, num_args=1, lru=True): return lambda orig: CacheDescriptor( orig, max_entries=max_entries, From 1e62a3d3a9d09fbd856b4abd733f4fb687f746e0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 Aug 2015 15:40:40 +0100 Subject: [PATCH 160/180] Up the cache size for 'get_joined_hosts_for_room' and 'get_users_in_room' --- synapse/storage/roommember.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 4db07f6fb..55dd3f6cf 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -78,7 +78,7 @@ class RoomMemberStore(SQLBaseStore): lambda events: events[0] if events else None ) - @cached() + @cached(max_entries=5000) def get_users_in_room(self, room_id): def f(txn): @@ -154,7 +154,7 @@ class RoomMemberStore(SQLBaseStore): RoomsForUser(**r) for r in self.cursor_to_dict(txn) ] - @cached() + @cached(max_entries=5000) def get_joined_hosts_for_room(self, room_id): return self.runInteraction( "get_joined_hosts_for_room", From 7eea3e356ff58168f3525879a8eb684f0681ee68 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Aug 2015 13:33:34 +0100 Subject: [PATCH 161/180] Make @cached cache deferreds rather than the deferreds' values --- synapse/storage/_base.py | 21 ++++++++------------- synapse/util/async.py | 9 +++++++-- tests/storage/test__base.py | 11 +++++++---- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f1265541b..8604d38c3 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -15,6 +15,7 @@ import logging from synapse.api.errors import StoreError +from synapse.util.async import ObservableDeferred from synapse.util.logutils import log_function from synapse.util.logcontext import preserve_context_over_fn, LoggingContext from synapse.util.lrucache import LruCache @@ -173,33 +174,27 @@ class CacheDescriptor(object): ) @functools.wraps(self.orig) - @defer.inlineCallbacks def wrapped(*args, **kwargs): arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs) keyargs = [arg_dict[arg_nm] for arg_nm in self.arg_names] try: cached_result = cache.get(*keyargs) - if DEBUG_CACHES: - actual_result = yield self.function_to_call(obj, *args, **kwargs) - if actual_result != cached_result: - logger.error( - "Stale cache entry %s%r: cached: %r, actual %r", - self.orig.__name__, keyargs, - cached_result, actual_result, - ) - raise ValueError("Stale cache entry") - defer.returnValue(cached_result) + return cached_result.observe() except KeyError: # Get the sequence number of the cache before reading from the # database so that we can tell if the cache is invalidated # while the SELECT is executing (SYN-369) sequence = cache.sequence - ret = yield self.function_to_call(obj, *args, **kwargs) + ret = defer.maybeDeferred( + self.function_to_call, + obj, *args, **kwargs + ) + ret = ObservableDeferred(ret, consumeErrors=False) cache.update(sequence, *(keyargs + [ret])) - defer.returnValue(ret) + return ret.observe() wrapped.invalidate = cache.invalidate wrapped.invalidate_all = cache.invalidate_all diff --git a/synapse/util/async.py b/synapse/util/async.py index 5a1d545c9..7bf2d38bb 100644 --- a/synapse/util/async.py +++ b/synapse/util/async.py @@ -51,7 +51,7 @@ class ObservableDeferred(object): object.__setattr__(self, "_observers", set()) def callback(r): - self._result = (True, r) + object.__setattr__(self, "_result", (True, r)) while self._observers: try: self._observers.pop().callback(r) @@ -60,7 +60,7 @@ class ObservableDeferred(object): return r def errback(f): - self._result = (False, f) + object.__setattr__(self, "_result", (False, f)) while self._observers: try: self._observers.pop().errback(f) @@ -97,3 +97,8 @@ class ObservableDeferred(object): def __setattr__(self, name, value): setattr(self._deferred, name, value) + + def __repr__(self): + return "" % ( + id(self), self._result, self._deferred, + ) diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index 8c3d2952b..8fa305d18 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -17,6 +17,8 @@ from tests import unittest from twisted.internet import defer +from synapse.util.async import ObservableDeferred + from synapse.storage._base import Cache, cached @@ -178,19 +180,20 @@ class CacheDecoratorTestCase(unittest.TestCase): self.assertTrue(callcount[0] >= 14, msg="Expected callcount >= 14, got %d" % (callcount[0])) - @defer.inlineCallbacks def test_prefill(self): callcount = [0] + d = defer.succeed(123) + class A(object): @cached() def func(self, key): callcount[0] += 1 - return key + return d a = A() - a.func.prefill("foo", 123) + a.func.prefill("foo", ObservableDeferred(d)) - self.assertEquals((yield a.func("foo")), 123) + self.assertEquals(a.func("foo").result, d.result) self.assertEquals(callcount[0], 0) From 433314cc34295ffefca349b9fc6914d81f2521e0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Aug 2015 14:01:05 +0100 Subject: [PATCH 162/180] Re-implement DEBUG_CACHES flag --- synapse/storage/_base.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 7b2be6745..8384299a1 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -178,8 +178,23 @@ class CacheDescriptor(object): arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs) keyargs = [arg_dict[arg_nm] for arg_nm in self.arg_names] try: - cached_result = cache.get(*keyargs) - return cached_result.observe() + cached_result_d = cache.get(*keyargs) + + if DEBUG_CACHES: + + @defer.inlineCallbacks + def check_result(cached_result): + actual_result = yield self.function_to_call(obj, *args, **kwargs) + if actual_result != cached_result: + logger.error( + "Stale cache entry %s%r: cached: %r, actual %r", + self.orig.__name__, keyargs, + cached_result, actual_result, + ) + raise ValueError("Stale cache entry") + cached_result_d.observe().addCallback(check_result) + + return cached_result_d.observe() except KeyError: # Get the sequence number of the cache before reading from the # database so that we can tell if the cache is invalidated From b811c9857491c0569ae367708721fbbaebf3adab Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Aug 2015 14:01:27 +0100 Subject: [PATCH 163/180] Remove failed deferreds from cache --- synapse/storage/_base.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 8384299a1..99c094875 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -205,8 +205,14 @@ class CacheDescriptor(object): self.function_to_call, obj, *args, **kwargs ) - ret = ObservableDeferred(ret, consumeErrors=False) + def onErr(f): + cache.invalidate(*keyargs) + return f + + ret.addErrback(onErr) + + ret = ObservableDeferred(ret, consumeErrors=False) cache.update(sequence, *(keyargs + [ret])) return ret.observe() From 63b1eaf32c57f0e1bae9b6a3768a560165637aee Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 Aug 2015 14:02:50 +0100 Subject: [PATCH 164/180] Docs --- synapse/storage/_base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 99c094875..0872a438f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -132,6 +132,9 @@ class Cache(object): class CacheDescriptor(object): """ A method decorator that applies a memoizing cache around the function. + This caches deferreds, rather than the results themselves. Deferreds that + fail are removed from the cache. + The function is presumed to take zero or more arguments, which are used in a tuple as the key for the cache. Hits are served directly from the cache; misses use the function body to generate the value. From efe60d5e8c76cede325aa09a138f5033b90b8d90 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 7 Aug 2015 16:36:42 +0100 Subject: [PATCH 165/180] Only print the pidfile path on startup if requested by a commandline flag --- synapse/app/homeserver.py | 3 ++- synapse/config/server.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 49e27c9e1..f04493f92 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -657,7 +657,8 @@ def run(hs): if hs.config.daemonize: - print hs.config.pid_file + if hs.config.print_pidfile: + print hs.config.pid_file daemon = Daemonize( app="synapse-homeserver", diff --git a/synapse/config/server.py b/synapse/config/server.py index f4d4a8710..f9a3b5f15 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -24,6 +24,7 @@ class ServerConfig(Config): self.web_client = config["web_client"] self.soft_file_limit = config["soft_file_limit"] self.daemonize = config.get("daemonize") + self.print_pidfile = config.get("print_pidfile") self.use_frozen_dicts = config.get("use_frozen_dicts", True) self.listeners = config.get("listeners", []) @@ -208,12 +209,18 @@ class ServerConfig(Config): self.manhole = args.manhole if args.daemonize is not None: self.daemonize = args.daemonize + if args.print_pidfile is not None: + self.print_pidfile = args.print_pidfile def add_arguments(self, parser): server_group = parser.add_argument_group("server") server_group.add_argument("-D", "--daemonize", action='store_true', default=None, help="Daemonize the home server") + server_group.add_argument("--print-pidfile", action='store_true', + default=None, + help="Print the path to the pidfile just" + " before daemonizing") server_group.add_argument("--manhole", metavar="PORT", dest="manhole", type=int, help="Turn on the twisted telnet manhole" From e3c8e2c13c7c99a10d52a984c0bffa5715868c9a Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 7 Aug 2015 16:42:27 +0100 Subject: [PATCH 166/180] Add a --generate-keys option --- synapse/config/_base.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index d483c67c6..c408db2b4 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -138,6 +138,11 @@ class Config(object): action="store_true", help="Generate a config file for the server name" ) + config_parser.add_argument( + "--generate-keys", + action="store_true", + help="Generate any missing key files then exit" + ) config_parser.add_argument( "-H", "--server-name", help="The server name to generate a config file for" @@ -230,4 +235,8 @@ class Config(object): obj.invoke_all("read_arguments", args) + if config_args.generate_keys: + obj.invoke_all("generate_files", config) + sys.exit(0) + return obj From 0db40d3e939506e22b927214bb0d3eaad40730ab Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 7 Aug 2015 17:22:11 +0100 Subject: [PATCH 167/180] Don't complain about extra .pyc files we find while hunting for database schemas --- synapse/storage/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 71d5d9250..973e78e04 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -354,6 +354,11 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files, ) logger.debug("Running script %s", relative_path) module.run_upgrade(cur, database_engine) + elif ext == ".pyc": + # Sometimes .pyc files turn up anyway even though we've + # disabled their generation; e.g. from distribution package + # installers. Silently skip it + pass elif ext == ".sql": # A plain old .sql file, just read and execute it logger.debug("Applying schema %s", relative_path) From 20addfa358e4f72b9f0c25d48c1e3ecfc08a68b2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Aug 2015 11:52:21 +0100 Subject: [PATCH 168/180] Change Cache to not use *args in its interface --- synapse/storage/__init__.py | 4 +- synapse/storage/_base.py | 61 +++++++++++++++-------------- synapse/storage/directory.py | 4 +- synapse/storage/event_federation.py | 4 +- synapse/storage/events.py | 21 +++++----- synapse/storage/keys.py | 2 +- synapse/storage/presence.py | 4 +- synapse/storage/push_rule.py | 16 ++++---- synapse/storage/registration.py | 2 +- synapse/storage/roommember.py | 6 +-- tests/storage/test__base.py | 12 +++--- 11 files changed, 69 insertions(+), 67 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 71d5d9250..1a6a8a376 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -99,7 +99,7 @@ class DataStore(RoomMemberStore, RoomStore, key = (user.to_string(), access_token, device_id, ip) try: - last_seen = self.client_ip_last_seen.get(*key) + last_seen = self.client_ip_last_seen.get(key) except KeyError: last_seen = None @@ -107,7 +107,7 @@ class DataStore(RoomMemberStore, RoomStore, if last_seen is not None and (now - last_seen) < LAST_SEEN_GRANULARITY: defer.returnValue(None) - self.client_ip_last_seen.prefill(*key + (now,)) + self.client_ip_last_seen.prefill(key, now) # It's safe not to lock here: a) no unique constraint, # b) LAST_SEEN_GRANULARITY makes concurrent updates incredibly unlikely diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 0872a438f..e07cf3b58 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -57,6 +57,9 @@ cache_counter = metrics.register_cache( ) +_CacheSentinel = object() + + class Cache(object): def __init__(self, name, max_entries=1000, keylen=1, lru=True): @@ -83,41 +86,38 @@ class Cache(object): "Cache objects can only be accessed from the main thread" ) - def get(self, *keyargs): - if len(keyargs) != self.keylen: - raise ValueError("Expected a key to have %d items", self.keylen) - - if keyargs in self.cache: + def get(self, keyargs, default=_CacheSentinel): + val = self.cache.get(keyargs, _CacheSentinel) + if val is not _CacheSentinel: cache_counter.inc_hits(self.name) - return self.cache[keyargs] + return val cache_counter.inc_misses(self.name) - raise KeyError() - def update(self, sequence, *args): + if default is _CacheSentinel: + raise KeyError() + else: + return default + + def update(self, sequence, keyargs, value): self.check_thread() if self.sequence == sequence: # Only update the cache if the caches sequence number matches the # number that the cache had before the SELECT was started (SYN-369) - self.prefill(*args) - - def prefill(self, *args): # because I can't *keyargs, value - keyargs = args[:-1] - value = args[-1] - - if len(keyargs) != self.keylen: - raise ValueError("Expected a key to have %d items", self.keylen) + self.prefill(keyargs, value) + def prefill(self, keyargs, value): if self.max_entries is not None: while len(self.cache) >= self.max_entries: self.cache.popitem(last=False) self.cache[keyargs] = value - def invalidate(self, *keyargs): + def invalidate(self, keyargs): self.check_thread() - if len(keyargs) != self.keylen: - raise ValueError("Expected a key to have %d items", self.keylen) + if not isinstance(keyargs, tuple): + raise ValueError("keyargs must be a tuple.") + # Increment the sequence number so that any SELECT statements that # raced with the INSERT don't update the cache (SYN-369) self.sequence += 1 @@ -168,20 +168,21 @@ class CacheDescriptor(object): % (orig.__name__,) ) - def __get__(self, obj, objtype=None): - cache = Cache( + self.cache = Cache( name=self.orig.__name__, max_entries=self.max_entries, keylen=self.num_args, lru=self.lru, ) + def __get__(self, obj, objtype=None): + @functools.wraps(self.orig) def wrapped(*args, **kwargs): arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs) - keyargs = [arg_dict[arg_nm] for arg_nm in self.arg_names] + keyargs = tuple(arg_dict[arg_nm] for arg_nm in self.arg_names) try: - cached_result_d = cache.get(*keyargs) + cached_result_d = self.cache.get(keyargs) if DEBUG_CACHES: @@ -202,7 +203,7 @@ class CacheDescriptor(object): # Get the sequence number of the cache before reading from the # database so that we can tell if the cache is invalidated # while the SELECT is executing (SYN-369) - sequence = cache.sequence + sequence = self.cache.sequence ret = defer.maybeDeferred( self.function_to_call, @@ -210,19 +211,19 @@ class CacheDescriptor(object): ) def onErr(f): - cache.invalidate(*keyargs) + self.cache.invalidate(keyargs) return f ret.addErrback(onErr) - ret = ObservableDeferred(ret, consumeErrors=False) - cache.update(sequence, *(keyargs + [ret])) + ret = ObservableDeferred(ret, consumeErrors=True) + self.cache.update(sequence, keyargs, ret) return ret.observe() - wrapped.invalidate = cache.invalidate - wrapped.invalidate_all = cache.invalidate_all - wrapped.prefill = cache.prefill + wrapped.invalidate = self.cache.invalidate + wrapped.invalidate_all = self.cache.invalidate_all + wrapped.prefill = self.cache.prefill obj.__dict__[self.orig.__name__] = wrapped diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 2b2bdf861..f3947bbe8 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -104,7 +104,7 @@ class DirectoryStore(SQLBaseStore): }, desc="create_room_alias_association", ) - self.get_aliases_for_room.invalidate(room_id) + self.get_aliases_for_room.invalidate((room_id,)) @defer.inlineCallbacks def delete_room_alias(self, room_alias): @@ -114,7 +114,7 @@ class DirectoryStore(SQLBaseStore): room_alias, ) - self.get_aliases_for_room.invalidate(room_id) + self.get_aliases_for_room.invalidate((room_id,)) defer.returnValue(room_id) def _delete_room_alias_txn(self, txn, room_alias): diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 45b86c94e..910b6598a 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -362,7 +362,7 @@ class EventFederationStore(SQLBaseStore): for room_id in events_by_room: txn.call_after( - self.get_latest_event_ids_in_room.invalidate, room_id + self.get_latest_event_ids_in_room.invalidate, (room_id,) ) def get_backfill_events(self, room_id, event_list, limit): @@ -505,4 +505,4 @@ class EventFederationStore(SQLBaseStore): query = "DELETE FROM event_forward_extremities WHERE room_id = ?" txn.execute(query, (room_id,)) - txn.call_after(self.get_latest_event_ids_in_room.invalidate, room_id) + txn.call_after(self.get_latest_event_ids_in_room.invalidate, (room_id,)) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index ed7ea3880..5b6491802 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -162,8 +162,8 @@ class EventsStore(SQLBaseStore): if current_state: txn.call_after(self.get_current_state_for_key.invalidate_all) txn.call_after(self.get_rooms_for_user.invalidate_all) - txn.call_after(self.get_users_in_room.invalidate, event.room_id) - txn.call_after(self.get_joined_hosts_for_room.invalidate, event.room_id) + txn.call_after(self.get_users_in_room.invalidate, (event.room_id,)) + txn.call_after(self.get_joined_hosts_for_room.invalidate, (event.room_id,)) txn.call_after(self.get_room_name_and_aliases, event.room_id) self._simple_delete_txn( @@ -430,13 +430,13 @@ class EventsStore(SQLBaseStore): if not context.rejected: txn.call_after( self.get_current_state_for_key.invalidate, - event.room_id, event.type, event.state_key - ) + (event.room_id, event.type, event.state_key,) + ) if event.type in [EventTypes.Name, EventTypes.Aliases]: txn.call_after( self.get_room_name_and_aliases.invalidate, - event.room_id + (event.room_id,) ) self._simple_upsert_txn( @@ -567,8 +567,9 @@ class EventsStore(SQLBaseStore): def _invalidate_get_event_cache(self, event_id): for check_redacted in (False, True): for get_prev_content in (False, True): - self._get_event_cache.invalidate(event_id, check_redacted, - get_prev_content) + self._get_event_cache.invalidate( + (event_id, check_redacted, get_prev_content) + ) def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=False, allow_rejected=False): @@ -589,7 +590,7 @@ class EventsStore(SQLBaseStore): for event_id in events: try: ret = self._get_event_cache.get( - event_id, check_redacted, get_prev_content + (event_id, check_redacted, get_prev_content,) ) if allow_rejected or not ret.rejected_reason: @@ -822,7 +823,7 @@ class EventsStore(SQLBaseStore): ev.unsigned["prev_content"] = prev.get_dict()["content"] self._get_event_cache.prefill( - ev.event_id, check_redacted, get_prev_content, ev + (ev.event_id, check_redacted, get_prev_content), ev ) defer.returnValue(ev) @@ -879,7 +880,7 @@ class EventsStore(SQLBaseStore): ev.unsigned["prev_content"] = prev.get_dict()["content"] self._get_event_cache.prefill( - ev.event_id, check_redacted, get_prev_content, ev + (ev.event_id, check_redacted, get_prev_content), ev ) return ev diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index e3f98f0cd..49b8e37cf 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -131,7 +131,7 @@ class KeyStore(SQLBaseStore): desc="store_server_verify_key", ) - self.get_all_server_verify_keys.invalidate(server_name) + self.get_all_server_verify_keys.invalidate((server_name,)) def store_server_keys_json(self, server_name, key_id, from_server, ts_now_ms, ts_expires_ms, key_json_bytes): diff --git a/synapse/storage/presence.py b/synapse/storage/presence.py index fefcf6bce..576cf670c 100644 --- a/synapse/storage/presence.py +++ b/synapse/storage/presence.py @@ -98,7 +98,7 @@ class PresenceStore(SQLBaseStore): updatevalues={"accepted": True}, desc="set_presence_list_accepted", ) - self.get_presence_list_accepted.invalidate(observer_localpart) + self.get_presence_list_accepted.invalidate((observer_localpart,)) defer.returnValue(result) def get_presence_list(self, observer_localpart, accepted=None): @@ -133,4 +133,4 @@ class PresenceStore(SQLBaseStore): "observed_user_id": observed_userid}, desc="del_presence_list", ) - self.get_presence_list_accepted.invalidate(observer_localpart) + self.get_presence_list_accepted.invalidate((observer_localpart,)) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index a220f3632..9b88ca7b3 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -151,11 +151,11 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, (user_name, priority_class, new_rule_priority)) txn.call_after( - self.get_push_rules_for_user.invalidate, user_name + self.get_push_rules_for_user.invalidate, (user_name,) ) txn.call_after( - self.get_push_rules_enabled_for_user.invalidate, user_name + self.get_push_rules_enabled_for_user.invalidate, (user_name,) ) self._simple_insert_txn( @@ -187,10 +187,10 @@ class PushRuleStore(SQLBaseStore): new_rule['priority'] = new_prio txn.call_after( - self.get_push_rules_for_user.invalidate, user_name + self.get_push_rules_for_user.invalidate, (user_name,) ) txn.call_after( - self.get_push_rules_enabled_for_user.invalidate, user_name + self.get_push_rules_enabled_for_user.invalidate, (user_name,) ) self._simple_insert_txn( @@ -216,8 +216,8 @@ class PushRuleStore(SQLBaseStore): desc="delete_push_rule", ) - self.get_push_rules_for_user.invalidate(user_name) - self.get_push_rules_enabled_for_user.invalidate(user_name) + self.get_push_rules_for_user.invalidate((user_name,)) + self.get_push_rules_enabled_for_user.invalidate((user_name,)) @defer.inlineCallbacks def set_push_rule_enabled(self, user_name, rule_id, enabled): @@ -238,10 +238,10 @@ class PushRuleStore(SQLBaseStore): {'id': new_id}, ) txn.call_after( - self.get_push_rules_for_user.invalidate, user_name + self.get_push_rules_for_user.invalidate, (user_name,) ) txn.call_after( - self.get_push_rules_enabled_for_user.invalidate, user_name + self.get_push_rules_enabled_for_user.invalidate, (user_name,) ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 90e2606be..4eaa088b3 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -131,7 +131,7 @@ class RegistrationStore(SQLBaseStore): user_id ) for r in rows: - self.get_user_by_token.invalidate(r) + self.get_user_by_token.invalidate((r,)) @cached() def get_user_by_token(self, token): diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 55dd3f6cf..9f14f38f2 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -54,9 +54,9 @@ class RoomMemberStore(SQLBaseStore): ) for event in events: - txn.call_after(self.get_rooms_for_user.invalidate, event.state_key) - txn.call_after(self.get_joined_hosts_for_room.invalidate, event.room_id) - txn.call_after(self.get_users_in_room.invalidate, event.room_id) + txn.call_after(self.get_rooms_for_user.invalidate, (event.state_key,)) + txn.call_after(self.get_joined_hosts_for_room.invalidate, (event.room_id,)) + txn.call_after(self.get_users_in_room.invalidate, (event.room_id,)) def get_room_member(self, user_id, room_id): """Retrieve the current state of a room member. diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index 8fa305d18..abee2f631 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -42,12 +42,12 @@ class CacheTestCase(unittest.TestCase): self.assertEquals(self.cache.get("foo"), 123) def test_invalidate(self): - self.cache.prefill("foo", 123) - self.cache.invalidate("foo") + self.cache.prefill(("foo",), 123) + self.cache.invalidate(("foo",)) failed = False try: - self.cache.get("foo") + self.cache.get(("foo",)) except KeyError: failed = True @@ -141,7 +141,7 @@ class CacheDecoratorTestCase(unittest.TestCase): self.assertEquals(callcount[0], 1) - a.func.invalidate("foo") + a.func.invalidate(("foo",)) yield a.func("foo") @@ -153,7 +153,7 @@ class CacheDecoratorTestCase(unittest.TestCase): def func(self, key): return key - A().func.invalidate("what") + A().func.invalidate(("what",)) @defer.inlineCallbacks def test_max_entries(self): @@ -193,7 +193,7 @@ class CacheDecoratorTestCase(unittest.TestCase): a = A() - a.func.prefill("foo", ObservableDeferred(d)) + a.func.prefill(("foo",), ObservableDeferred(d)) self.assertEquals(a.func("foo").result, d.result) self.assertEquals(callcount[0], 0) From 3213ff630ca6f7f624d65dbab55e2c4c65fe80ce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Aug 2015 19:13:45 +0100 Subject: [PATCH 169/180] Remove unnecessary cache --- synapse/storage/state.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 55c6d5289..7ce51b9bd 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore, cached, cachedInlineCallbacks +from ._base import SQLBaseStore, cachedInlineCallbacks from twisted.internet import defer @@ -91,7 +91,6 @@ class StateStore(SQLBaseStore): defer.returnValue(dict(state_list)) - @cached(num_args=1) def _fetch_events_for_group(self, key, events): return self._get_events( events, get_prev_content=False From 62126c996c5bc8d86d3e15c015a68e8e60622a55 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Aug 2015 19:17:58 +0100 Subject: [PATCH 170/180] Propogate stale cache errors to calling functions --- synapse/storage/_base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 0872a438f..524a00315 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -183,8 +183,8 @@ class CacheDescriptor(object): try: cached_result_d = cache.get(*keyargs) + observed = cached_result_d.observe() if DEBUG_CACHES: - @defer.inlineCallbacks def check_result(cached_result): actual_result = yield self.function_to_call(obj, *args, **kwargs) @@ -195,9 +195,10 @@ class CacheDescriptor(object): cached_result, actual_result, ) raise ValueError("Stale cache entry") - cached_result_d.observe().addCallback(check_result) + defer.returnValue(cached_result) + observed.addCallback(check_result) - return cached_result_d.observe() + return observed except KeyError: # Get the sequence number of the cache before reading from the # database so that we can tell if the cache is invalidated From 9c5385b53ad6d8ac21227dbc3f150c7d2511b061 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Aug 2015 19:26:38 +0100 Subject: [PATCH 171/180] s/observed/observer/ --- synapse/storage/_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 524a00315..5997603b3 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -183,7 +183,7 @@ class CacheDescriptor(object): try: cached_result_d = cache.get(*keyargs) - observed = cached_result_d.observe() + observer = cached_result_d.observe() if DEBUG_CACHES: @defer.inlineCallbacks def check_result(cached_result): @@ -196,9 +196,9 @@ class CacheDescriptor(object): ) raise ValueError("Stale cache entry") defer.returnValue(cached_result) - observed.addCallback(check_result) + observer.addCallback(check_result) - return observed + return observer except KeyError: # Get the sequence number of the cache before reading from the # database so that we can tell if the cache is invalidated From 2cd6cb9f65e64ea49e0a8ba7c4851c79821243ad Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Aug 2015 10:38:47 +0100 Subject: [PATCH 172/180] Rename keyargs to args in Cache --- synapse/storage/_base.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a088a6175..1e2d43666 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -86,8 +86,8 @@ class Cache(object): "Cache objects can only be accessed from the main thread" ) - def get(self, keyargs, default=_CacheSentinel): - val = self.cache.get(keyargs, _CacheSentinel) + def get(self, key, default=_CacheSentinel): + val = self.cache.get(key, _CacheSentinel) if val is not _CacheSentinel: cache_counter.inc_hits(self.name) return val @@ -99,29 +99,29 @@ class Cache(object): else: return default - def update(self, sequence, keyargs, value): + def update(self, sequence, key, value): self.check_thread() if self.sequence == sequence: # Only update the cache if the caches sequence number matches the # number that the cache had before the SELECT was started (SYN-369) - self.prefill(keyargs, value) + self.prefill(key, value) - def prefill(self, keyargs, value): + def prefill(self, key, value): if self.max_entries is not None: while len(self.cache) >= self.max_entries: self.cache.popitem(last=False) - self.cache[keyargs] = value + self.cache[key] = value - def invalidate(self, keyargs): + def invalidate(self, key): self.check_thread() - if not isinstance(keyargs, tuple): + if not isinstance(key, tuple): raise ValueError("keyargs must be a tuple.") # Increment the sequence number so that any SELECT statements that # raced with the INSERT don't update the cache (SYN-369) self.sequence += 1 - self.cache.pop(keyargs, None) + self.cache.pop(key, None) def invalidate_all(self): self.check_thread() From 86eaaa885bb4f3d75990874a76c53b2a0870d639 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 Aug 2015 13:44:44 +0100 Subject: [PATCH 173/180] Rename keyargs to args in CacheDescriptor --- synapse/storage/_base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1e2d43666..e76ee2779 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -180,9 +180,9 @@ class CacheDescriptor(object): @functools.wraps(self.orig) def wrapped(*args, **kwargs): arg_dict = inspect.getcallargs(self.orig, obj, *args, **kwargs) - keyargs = tuple(arg_dict[arg_nm] for arg_nm in self.arg_names) + cache_key = tuple(arg_dict[arg_nm] for arg_nm in self.arg_names) try: - cached_result_d = self.cache.get(keyargs) + cached_result_d = self.cache.get(cache_key) observer = cached_result_d.observe() if DEBUG_CACHES: @@ -192,7 +192,7 @@ class CacheDescriptor(object): if actual_result != cached_result: logger.error( "Stale cache entry %s%r: cached: %r, actual %r", - self.orig.__name__, keyargs, + self.orig.__name__, cache_key, cached_result, actual_result, ) raise ValueError("Stale cache entry") @@ -212,13 +212,13 @@ class CacheDescriptor(object): ) def onErr(f): - self.cache.invalidate(keyargs) + self.cache.invalidate(cache_key) return f ret.addErrback(onErr) ret = ObservableDeferred(ret, consumeErrors=True) - self.cache.update(sequence, keyargs, ret) + self.cache.update(sequence, cache_key, ret) return ret.observe() From 559c51debcf5402cd5999aa1afb854b34e9de363 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 10 Aug 2015 14:07:08 +0100 Subject: [PATCH 174/180] Use TypeError instead of ValueError and give a nicer error mesasge when someone calls Cache.invalidate with the wrong type. --- synapse/storage/_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e76ee2779..73eea157a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -116,7 +116,9 @@ class Cache(object): def invalidate(self, key): self.check_thread() if not isinstance(key, tuple): - raise ValueError("keyargs must be a tuple.") + raise TypeError( + "The cache key must be a tuple not %r" % (type(key),) + ) # Increment the sequence number so that any SELECT statements that # raced with the INSERT don't update the cache (SYN-369) From 9dba813234fdeeef5097572dec9c975a4a83a99b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 11 Aug 2015 16:34:17 +0100 Subject: [PATCH 175/180] Remove redundant if-guard The startswith("@") does the job --- synapse/api/auth.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 487be7ce9..bf02347f6 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -521,23 +521,22 @@ class Auth(object): # Check state_key if hasattr(event, "state_key"): - if not event.state_key.startswith("_"): - if event.state_key.startswith("@"): - if event.state_key != event.user_id: + if event.state_key.startswith("@"): + if event.state_key != event.user_id: + raise AuthError( + 403, + "You are not allowed to set others state" + ) + else: + sender_domain = UserID.from_string( + event.user_id + ).domain + + if sender_domain != event.state_key: raise AuthError( 403, "You are not allowed to set others state" ) - else: - sender_domain = UserID.from_string( - event.user_id - ).domain - - if sender_domain != event.state_key: - raise AuthError( - 403, - "You are not allowed to set others state" - ) return True From 88e03da39f3746f7d754150964da064772b6ba5f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 11 Aug 2015 16:35:28 +0100 Subject: [PATCH 176/180] Minor docs cleanup --- synapse/api/auth.py | 9 +++++++-- synapse/types.py | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index bf02347f6..0eef7442e 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -44,6 +44,11 @@ class Auth(object): def check(self, event, auth_events): """ Checks if this event is correctly authed. + Args: + event: the event being checked. + auth_events (dict: event-key -> event): the existing room state. + + Returns: True if the auth checks pass. """ @@ -319,7 +324,7 @@ class Auth(object): Returns: tuple : of UserID and device string: User ID object of the user making the request - Client ID object of the client instance the user is using + ClientInfo object of the client instance the user is using Raises: AuthError if no user by that token exists or the token is invalid. """ @@ -352,7 +357,7 @@ class Auth(object): ) return except KeyError: - pass # normal users won't have this query parameter set + pass # normal users won't have the user_id query parameter set. user_info = yield self.get_user_by_token(access_token) user = user_info["user"] diff --git a/synapse/types.py b/synapse/types.py index dd1b10d64..e190374cb 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -178,7 +178,7 @@ class RoomStreamToken(namedtuple("_StreamToken", "topological stream")): Live tokens start with an "s" followed by the "stream_ordering" id of the event it comes after. Historic tokens start with a "t" followed by the - "topological_ordering" id of the event it comes after, follewed by "-", + "topological_ordering" id of the event it comes after, followed by "-", followed by the "stream_ordering" id of the event it comes after. """ __slots__ = [] @@ -211,4 +211,5 @@ class RoomStreamToken(namedtuple("_StreamToken", "topological stream")): return "s%d" % (self.stream,) +# token_id is the primary key ID of the access token, not the access token itself. ClientInfo = namedtuple("ClientInfo", ("device_id", "token_id")) From 45610305eac1d5ce15faf0bf0482442aa896f4a6 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 11 Aug 2015 16:43:27 +0100 Subject: [PATCH 177/180] Add missing space because linter --- synapse/api/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 0eef7442e..a7f428a96 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -357,7 +357,7 @@ class Auth(object): ) return except KeyError: - pass # normal users won't have the user_id query parameter set. + pass # normal users won't have the user_id query parameter set. user_info = yield self.get_user_by_token(access_token) user = user_info["user"] From 4ff0228c254e146e37ffd2b1c1df22d1b67353cb Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 11 Aug 2015 16:54:06 +0100 Subject: [PATCH 178/180] Remove call to recently removed function in mock --- tests/test_distributor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_distributor.py b/tests/test_distributor.py index 6a0095d85..8ed48cfb7 100644 --- a/tests/test_distributor.py +++ b/tests/test_distributor.py @@ -73,8 +73,8 @@ class DistributorTestCase(unittest.TestCase): yield d self.assertTrue(d.called) - observers[0].assert_called_once("Go") - observers[1].assert_called_once("Go") + observers[0].assert_called_once_with("Go") + observers[1].assert_called_once_with("Go") self.assertEquals(mock_logger.warning.call_count, 1) self.assertIsInstance(mock_logger.warning.call_args[0][0], From b24d7ebd6e57154d13f3a3a8e1aacc20ef34b417 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 12 Aug 2015 09:39:07 +0100 Subject: [PATCH 179/180] Fix the cleanup script to use the right $DIR --- demo/clean.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demo/clean.sh b/demo/clean.sh index 31007737f..418ca9457 100755 --- a/demo/clean.sh +++ b/demo/clean.sh @@ -12,8 +12,8 @@ if [ -f $PID_FILE ]; then fi for port in 8080 8081 8082; do - rm -rf demo/$port - rm -rf demo/media_store.$port + rm -rf $DIR/$port + rm -rf $DIR/media_store.$port done rm -rf $DIR/etc From 7bbaab94327d806617fc136aab51303baed060cf Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 12 Aug 2015 11:57:37 +0100 Subject: [PATCH 180/180] Fix the --generate-keys option. Make it do the same thing as --generate-config does when the config file exists, but without printing a warning --- synapse/config/_base.py | 83 ++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 46 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index c408db2b4..73f695995 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -149,6 +149,8 @@ class Config(object): ) config_args, remaining_args = config_parser.parse_known_args(argv) + generate_keys = config_args.generate_keys + if config_args.generate_config: if not config_args.config_path: config_parser.error( @@ -156,51 +158,40 @@ class Config(object): " generated using \"--generate-config -H SERVER_NAME" " -c CONFIG-FILE\"" ) - - config_dir_path = os.path.dirname(config_args.config_path[0]) - config_dir_path = os.path.abspath(config_dir_path) - - server_name = config_args.server_name - if not server_name: - print "Must specify a server_name to a generate config for." - sys.exit(1) (config_path,) = config_args.config_path - if not os.path.exists(config_dir_path): - os.makedirs(config_dir_path) - if os.path.exists(config_path): - print "Config file %r already exists" % (config_path,) - yaml_config = cls.read_config_file(config_path) - yaml_name = yaml_config["server_name"] - if server_name != yaml_name: - print ( - "Config file %r has a different server_name: " - " %r != %r" % (config_path, server_name, yaml_name) - ) + if not os.path.exists(config_path): + config_dir_path = os.path.dirname(config_path) + config_dir_path = os.path.abspath(config_dir_path) + + server_name = config_args.server_name + if not server_name: + print "Must specify a server_name to a generate config for." sys.exit(1) - config_bytes, config = obj.generate_config( - config_dir_path, server_name - ) - config.update(yaml_config) - print "Generating any missing keys for %r" % (server_name,) - obj.invoke_all("generate_files", config) - sys.exit(0) - with open(config_path, "wb") as config_file: - config_bytes, config = obj.generate_config( - config_dir_path, server_name - ) - obj.invoke_all("generate_files", config) - config_file.write(config_bytes) + if not os.path.exists(config_dir_path): + os.makedirs(config_dir_path) + with open(config_path, "wb") as config_file: + config_bytes, config = obj.generate_config( + config_dir_path, server_name + ) + obj.invoke_all("generate_files", config) + config_file.write(config_bytes) print ( - "A config file has been generated in %s for server name" - " '%s' with corresponding SSL keys and self-signed" - " certificates. Please review this file and customise it to" - " your needs." + "A config file has been generated in %r for server name" + " %r with corresponding SSL keys and self-signed" + " certificates. Please review this file and customise it" + " to your needs." ) % (config_path, server_name) - print ( - "If this server name is incorrect, you will need to regenerate" - " the SSL certificates" - ) - sys.exit(0) + print ( + "If this server name is incorrect, you will need to" + " regenerate the SSL certificates" + ) + sys.exit(0) + else: + print ( + "Config file %r already exists. Generating any missing key" + " files." + ) % (config_path,) + generate_keys = True parser = argparse.ArgumentParser( parents=[config_parser], @@ -218,7 +209,7 @@ class Config(object): " -c CONFIG-FILE\"" ) - config_dir_path = os.path.dirname(config_args.config_path[0]) + config_dir_path = os.path.dirname(config_args.config_path[-1]) config_dir_path = os.path.abspath(config_dir_path) specified_config = {} @@ -231,12 +222,12 @@ class Config(object): config.pop("log_config") config.update(specified_config) + if generate_keys: + obj.invoke_all("generate_files", config) + sys.exit(0) + obj.invoke_all("read_config", config) obj.invoke_all("read_arguments", args) - if config_args.generate_keys: - obj.invoke_all("generate_files", config) - sys.exit(0) - return obj